From f61537a8c5ec685e9000b21ee9c830dd1c0622ef Mon Sep 17 00:00:00 2001 From: Xubin Ren <52506698+Re-bin@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:08:10 +0900 Subject: [PATCH] fix(tools): preserve computer use runtime state --- nanobot/agent/tools/browser_tool.py | 18 ++++++++++----- nanobot/agent/tools/computer_use.py | 22 +++++++++++------- nanobot/webui/settings_routes.py | 10 +++++++-- tests/tools/test_browser_tool.py | 2 +- tests/tools/test_computer_use_tool.py | 2 +- tests/webui/test_settings_routes.py | 32 ++++++++++++++++++++++++--- 6 files changed, 65 insertions(+), 21 deletions(-) diff --git a/nanobot/agent/tools/browser_tool.py b/nanobot/agent/tools/browser_tool.py index cf2f18b59..5f514996a 100644 --- a/nanobot/agent/tools/browser_tool.py +++ b/nanobot/agent/tools/browser_tool.py @@ -9,7 +9,7 @@ from typing import Any from pydantic import Field -from nanobot.agent.tools.base import Tool, tool_parameters +from nanobot.agent.tools.base import Tool, ToolResult, tool_parameters from nanobot.agent.tools.computer_use_backends.base import SessionBackendPool from nanobot.agent.tools.context import ToolContext from nanobot.agent.tools.schema import ( @@ -235,23 +235,29 @@ class BrowserTool(Tool): async def _execute(self, action: str | None = None, **kwargs: Any) -> Any: action = (action or "").strip() if action not in _ACTIONS: - return f"Error: unknown action '{action}'. Valid actions: {', '.join(_ACTIONS)}" + return ToolResult.error( + f"Error: unknown action '{action}'. Valid actions: {', '.join(_ACTIONS)}" + ) try: backend = await self._backends.get() except ImportError as exc: - return f"Error: {exc}" + return ToolResult.error(f"Error: {exc}") except Exception as exc: - return f"Error: could not initialize browser backend: {type(exc).__name__}: {exc}" + return ToolResult.error( + f"Error: could not initialize browser backend: {type(exc).__name__}: {exc}" + ) try: status, direct = await self._dispatch(backend, action, kwargs) if blocked := getattr(backend, "pop_blocked_navigation", lambda: None)(): raise ValueError(f"navigation was blocked: {blocked}") except ValueError as exc: - return f"Error: {exc}" + return ToolResult.error(f"Error: {exc}") except Exception as exc: - return f"Error executing browser '{action}': {type(exc).__name__}: {exc}" + return ToolResult.error( + f"Error executing browser '{action}': {type(exc).__name__}: {exc}" + ) if direct is not None: return direct diff --git a/nanobot/agent/tools/computer_use.py b/nanobot/agent/tools/computer_use.py index a60d5ffa2..98117ebc5 100644 --- a/nanobot/agent/tools/computer_use.py +++ b/nanobot/agent/tools/computer_use.py @@ -10,7 +10,7 @@ from typing import Any, Literal from pydantic import Field -from nanobot.agent.tools.base import Tool, tool_parameters +from nanobot.agent.tools.base import Tool, ToolResult, tool_parameters from nanobot.agent.tools.computer_use_backends.base import SessionBackendPool from nanobot.agent.tools.context import ToolContext from nanobot.agent.tools.schema import ( @@ -287,15 +287,19 @@ class ComputerUseTool(Tool): async def _execute(self, action: str | None = None, **kwargs: Any) -> Any: action = (action or "").strip() if action not in _ACTIONS: - return f"Error: unknown action '{action}'. Valid actions: {', '.join(_ACTIONS)}" + return ToolResult.error( + f"Error: unknown action '{action}'. Valid actions: {', '.join(_ACTIONS)}" + ) try: backend = await self._backends.get() real_w, real_h = await backend.dimensions() except ImportError as exc: - return f"Error: {exc}" + return ToolResult.error(f"Error: {exc}") except Exception as exc: - return f"Error: could not initialize computer_use backend: {type(exc).__name__}: {exc}" + return ToolResult.error( + f"Error: could not initialize computer_use backend: {type(exc).__name__}: {exc}" + ) source = (real_w, real_h) target = _fit_size(real_w, real_h, self.config.target_width, self.config.target_height) @@ -305,18 +309,20 @@ class ComputerUseTool(Tool): if blocked := getattr(backend, "pop_blocked_navigation", lambda: None)(): raise ValueError(f"navigation was blocked: {blocked}") except ValueError as exc: - return f"Error: {exc}" + return ToolResult.error(f"Error: {exc}") except NotImplementedError as exc: - return f"Error: {exc}" + return ToolResult.error(f"Error: {exc}") except Exception as exc: - return f"Error executing computer_use '{action}': {type(exc).__name__}: {exc}" + return ToolResult.error( + f"Error executing computer_use '{action}': {type(exc).__name__}: {exc}" + ) # Return a fresh screenshot so the model sees the result of its action. try: png = await backend.screenshot() png = self._downscale_png(png, target) except ImportError as exc: - return f"Error: {exc}" + return ToolResult.error(f"Error: {exc}") except Exception as exc: return f"{status}\n(Could not capture screenshot: {type(exc).__name__}: {exc})" diff --git a/nanobot/webui/settings_routes.py b/nanobot/webui/settings_routes.py index b2a8f8def..38d8c2198 100644 --- a/nanobot/webui/settings_routes.py +++ b/nanobot/webui/settings_routes.py @@ -512,11 +512,17 @@ class WebUISettingsRouter: def _handle_settings_computer_use_update(self, request: WsRequest) -> Response: if not self._authorized(request): return self._unauthorized() + query = self._query(request) try: - payload = update_computer_use_settings(self._query(request)) + payload = update_computer_use_settings(query) except WebUISettingsError as e: return self._error_response(e.status, e.message) - return self._json_response(self._with_restart_state(payload, section="runtime")) + if payload.get("requires_restart"): + if "browser_enabled" in query or "browserEnabled" in query: + self._restart_sections.add("browser") + if "enabled" in query or "computerEnabled" in query: + self._restart_sections.add("runtime") + return self._json_response(self._with_restart_state(payload)) def _handle_settings_api_service(self, request: WsRequest) -> Response: if not self._authorized(request): diff --git a/tests/tools/test_browser_tool.py b/tests/tools/test_browser_tool.py index a5bc92f10..f84a7420b 100644 --- a/tests/tools/test_browser_tool.py +++ b/tests/tools/test_browser_tool.py @@ -130,7 +130,6 @@ class TestDispatch: assert '[1] button "Submit"' in result assert '[2] input[text] "your name"' in result - @pytest.mark.asyncio @pytest.mark.parametrize( ("action", "kwargs", "expected"), [ @@ -214,6 +213,7 @@ class TestErrorsAndPolicy: tool, _ = _tool() result = await tool.execute(**kwargs) assert isinstance(result, str) and error in result + assert result.is_error is True @pytest.mark.asyncio async def test_backend_blocks_disallowed_navigation(self): diff --git a/tests/tools/test_computer_use_tool.py b/tests/tools/test_computer_use_tool.py index fe2c49de4..c2bc553d5 100644 --- a/tests/tools/test_computer_use_tool.py +++ b/tests/tools/test_computer_use_tool.py @@ -161,7 +161,6 @@ class TestExecute: await tool.execute(action="left_click", x=5000, y=-10) assert fb.calls == [("click", 2559, 0, "left", 1)] - @pytest.mark.asyncio @pytest.mark.parametrize( ("action", "kwargs", "expected"), [ @@ -206,6 +205,7 @@ class TestExecute: tool, _ = _tool() result = await tool.execute(**kwargs) assert isinstance(result, str) and error in result + assert result.is_error is True @pytest.mark.asyncio diff --git a/tests/webui/test_settings_routes.py b/tests/webui/test_settings_routes.py index a0573d26f..fc8c7ac7b 100644 --- a/tests/webui/test_settings_routes.py +++ b/tests/webui/test_settings_routes.py @@ -140,8 +140,33 @@ async def test_model_preset_mutation_routes( assert captured["query"] == expected_query +@pytest.mark.parametrize( + ("query", "expected_query", "expected_sections"), + [ + ( + "browser_enabled=true", + {"browser_enabled": ["true"]}, + ["browser"], + ), + ( + "computerEnabled=true", + {"computerEnabled": ["true"]}, + ["runtime"], + ), + ( + "browserEnabled=true&enabled=true", + {"browserEnabled": ["true"], "enabled": ["true"]}, + ["browser", "runtime"], + ), + ], +) @pytest.mark.asyncio -async def test_computer_use_update_route(monkeypatch) -> None: +async def test_computer_use_update_route( + monkeypatch, + query: str, + expected_query: dict[str, list[str]], + expected_sections: list[str], +) -> None: captured: dict[str, object] = {} def update(query): @@ -150,7 +175,7 @@ async def test_computer_use_update_route(monkeypatch) -> None: monkeypatch.setattr("nanobot.webui.settings_routes.update_computer_use_settings", update) request = SimpleNamespace( - path="/api/settings/computer-use/update?browser_enabled=true", + path=f"/api/settings/computer-use/update?{query}", headers=Headers(), ) @@ -162,4 +187,5 @@ async def test_computer_use_update_route(monkeypatch) -> None: assert response is not None assert response.status_code == 200 - assert captured["query"] == {"browser_enabled": ["true"]} + assert captured["query"] == expected_query + assert json.loads(response.body)["restart_required_sections"] == expected_sections