mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-08-11 14:58:39 +03:00
fix(tools): preserve computer use runtime state
This commit is contained in:
@@ -9,7 +9,7 @@ from typing import Any
|
|||||||
|
|
||||||
from pydantic import Field
|
from pydantic import Field
|
||||||
|
|
||||||
from nanobot.agent.tools.base import Tool, tool_parameters
|
from nanobot.agent.tools.base import Tool, ToolResult, tool_parameters
|
||||||
from nanobot.agent.tools.computer_use_backends.base import SessionBackendPool
|
from nanobot.agent.tools.computer_use_backends.base import SessionBackendPool
|
||||||
from nanobot.agent.tools.context import ToolContext
|
from nanobot.agent.tools.context import ToolContext
|
||||||
from nanobot.agent.tools.schema import (
|
from nanobot.agent.tools.schema import (
|
||||||
@@ -235,23 +235,29 @@ class BrowserTool(Tool):
|
|||||||
async def _execute(self, action: str | None = None, **kwargs: Any) -> Any:
|
async def _execute(self, action: str | None = None, **kwargs: Any) -> Any:
|
||||||
action = (action or "").strip()
|
action = (action or "").strip()
|
||||||
if action not in _ACTIONS:
|
if action not in _ACTIONS:
|
||||||
return f"Error: unknown action '{action}'. Valid actions: {', '.join(_ACTIONS)}"
|
return ToolResult.error(
|
||||||
|
f"Error: unknown action '{action}'. Valid actions: {', '.join(_ACTIONS)}"
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
backend = await self._backends.get()
|
backend = await self._backends.get()
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
return f"Error: {exc}"
|
return ToolResult.error(f"Error: {exc}")
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return f"Error: could not initialize browser backend: {type(exc).__name__}: {exc}"
|
return ToolResult.error(
|
||||||
|
f"Error: could not initialize browser backend: {type(exc).__name__}: {exc}"
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
status, direct = await self._dispatch(backend, action, kwargs)
|
status, direct = await self._dispatch(backend, action, kwargs)
|
||||||
if blocked := getattr(backend, "pop_blocked_navigation", lambda: None)():
|
if blocked := getattr(backend, "pop_blocked_navigation", lambda: None)():
|
||||||
raise ValueError(f"navigation was blocked: {blocked}")
|
raise ValueError(f"navigation was blocked: {blocked}")
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return f"Error: {exc}"
|
return ToolResult.error(f"Error: {exc}")
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return f"Error executing browser '{action}': {type(exc).__name__}: {exc}"
|
return ToolResult.error(
|
||||||
|
f"Error executing browser '{action}': {type(exc).__name__}: {exc}"
|
||||||
|
)
|
||||||
|
|
||||||
if direct is not None:
|
if direct is not None:
|
||||||
return direct
|
return direct
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from typing import Any, Literal
|
|||||||
|
|
||||||
from pydantic import Field
|
from pydantic import Field
|
||||||
|
|
||||||
from nanobot.agent.tools.base import Tool, tool_parameters
|
from nanobot.agent.tools.base import Tool, ToolResult, tool_parameters
|
||||||
from nanobot.agent.tools.computer_use_backends.base import SessionBackendPool
|
from nanobot.agent.tools.computer_use_backends.base import SessionBackendPool
|
||||||
from nanobot.agent.tools.context import ToolContext
|
from nanobot.agent.tools.context import ToolContext
|
||||||
from nanobot.agent.tools.schema import (
|
from nanobot.agent.tools.schema import (
|
||||||
@@ -287,15 +287,19 @@ class ComputerUseTool(Tool):
|
|||||||
async def _execute(self, action: str | None = None, **kwargs: Any) -> Any:
|
async def _execute(self, action: str | None = None, **kwargs: Any) -> Any:
|
||||||
action = (action or "").strip()
|
action = (action or "").strip()
|
||||||
if action not in _ACTIONS:
|
if action not in _ACTIONS:
|
||||||
return f"Error: unknown action '{action}'. Valid actions: {', '.join(_ACTIONS)}"
|
return ToolResult.error(
|
||||||
|
f"Error: unknown action '{action}'. Valid actions: {', '.join(_ACTIONS)}"
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
backend = await self._backends.get()
|
backend = await self._backends.get()
|
||||||
real_w, real_h = await backend.dimensions()
|
real_w, real_h = await backend.dimensions()
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
return f"Error: {exc}"
|
return ToolResult.error(f"Error: {exc}")
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return f"Error: could not initialize computer_use backend: {type(exc).__name__}: {exc}"
|
return ToolResult.error(
|
||||||
|
f"Error: could not initialize computer_use backend: {type(exc).__name__}: {exc}"
|
||||||
|
)
|
||||||
|
|
||||||
source = (real_w, real_h)
|
source = (real_w, real_h)
|
||||||
target = _fit_size(real_w, real_h, self.config.target_width, self.config.target_height)
|
target = _fit_size(real_w, real_h, self.config.target_width, self.config.target_height)
|
||||||
@@ -305,18 +309,20 @@ class ComputerUseTool(Tool):
|
|||||||
if blocked := getattr(backend, "pop_blocked_navigation", lambda: None)():
|
if blocked := getattr(backend, "pop_blocked_navigation", lambda: None)():
|
||||||
raise ValueError(f"navigation was blocked: {blocked}")
|
raise ValueError(f"navigation was blocked: {blocked}")
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return f"Error: {exc}"
|
return ToolResult.error(f"Error: {exc}")
|
||||||
except NotImplementedError as exc:
|
except NotImplementedError as exc:
|
||||||
return f"Error: {exc}"
|
return ToolResult.error(f"Error: {exc}")
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return f"Error executing computer_use '{action}': {type(exc).__name__}: {exc}"
|
return ToolResult.error(
|
||||||
|
f"Error executing computer_use '{action}': {type(exc).__name__}: {exc}"
|
||||||
|
)
|
||||||
|
|
||||||
# Return a fresh screenshot so the model sees the result of its action.
|
# Return a fresh screenshot so the model sees the result of its action.
|
||||||
try:
|
try:
|
||||||
png = await backend.screenshot()
|
png = await backend.screenshot()
|
||||||
png = self._downscale_png(png, target)
|
png = self._downscale_png(png, target)
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
return f"Error: {exc}"
|
return ToolResult.error(f"Error: {exc}")
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return f"{status}\n(Could not capture screenshot: {type(exc).__name__}: {exc})"
|
return f"{status}\n(Could not capture screenshot: {type(exc).__name__}: {exc})"
|
||||||
|
|
||||||
|
|||||||
@@ -512,11 +512,17 @@ class WebUISettingsRouter:
|
|||||||
def _handle_settings_computer_use_update(self, request: WsRequest) -> Response:
|
def _handle_settings_computer_use_update(self, request: WsRequest) -> Response:
|
||||||
if not self._authorized(request):
|
if not self._authorized(request):
|
||||||
return self._unauthorized()
|
return self._unauthorized()
|
||||||
|
query = self._query(request)
|
||||||
try:
|
try:
|
||||||
payload = update_computer_use_settings(self._query(request))
|
payload = update_computer_use_settings(query)
|
||||||
except WebUISettingsError as e:
|
except WebUISettingsError as e:
|
||||||
return self._error_response(e.status, e.message)
|
return self._error_response(e.status, e.message)
|
||||||
return self._json_response(self._with_restart_state(payload, section="runtime"))
|
if payload.get("requires_restart"):
|
||||||
|
if "browser_enabled" in query or "browserEnabled" in query:
|
||||||
|
self._restart_sections.add("browser")
|
||||||
|
if "enabled" in query or "computerEnabled" in query:
|
||||||
|
self._restart_sections.add("runtime")
|
||||||
|
return self._json_response(self._with_restart_state(payload))
|
||||||
|
|
||||||
def _handle_settings_api_service(self, request: WsRequest) -> Response:
|
def _handle_settings_api_service(self, request: WsRequest) -> Response:
|
||||||
if not self._authorized(request):
|
if not self._authorized(request):
|
||||||
|
|||||||
@@ -130,7 +130,6 @@ class TestDispatch:
|
|||||||
assert '[1] button "Submit"' in result
|
assert '[1] button "Submit"' in result
|
||||||
assert '[2] input[text] "your name"' in result
|
assert '[2] input[text] "your name"' in result
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("action", "kwargs", "expected"),
|
("action", "kwargs", "expected"),
|
||||||
[
|
[
|
||||||
@@ -214,6 +213,7 @@ class TestErrorsAndPolicy:
|
|||||||
tool, _ = _tool()
|
tool, _ = _tool()
|
||||||
result = await tool.execute(**kwargs)
|
result = await tool.execute(**kwargs)
|
||||||
assert isinstance(result, str) and error in result
|
assert isinstance(result, str) and error in result
|
||||||
|
assert result.is_error is True
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_backend_blocks_disallowed_navigation(self):
|
async def test_backend_blocks_disallowed_navigation(self):
|
||||||
|
|||||||
@@ -161,7 +161,6 @@ class TestExecute:
|
|||||||
await tool.execute(action="left_click", x=5000, y=-10)
|
await tool.execute(action="left_click", x=5000, y=-10)
|
||||||
assert fb.calls == [("click", 2559, 0, "left", 1)]
|
assert fb.calls == [("click", 2559, 0, "left", 1)]
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("action", "kwargs", "expected"),
|
("action", "kwargs", "expected"),
|
||||||
[
|
[
|
||||||
@@ -206,6 +205,7 @@ class TestExecute:
|
|||||||
tool, _ = _tool()
|
tool, _ = _tool()
|
||||||
result = await tool.execute(**kwargs)
|
result = await tool.execute(**kwargs)
|
||||||
assert isinstance(result, str) and error in result
|
assert isinstance(result, str) and error in result
|
||||||
|
assert result.is_error is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -140,8 +140,33 @@ async def test_model_preset_mutation_routes(
|
|||||||
assert captured["query"] == expected_query
|
assert captured["query"] == expected_query
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("query", "expected_query", "expected_sections"),
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"browser_enabled=true",
|
||||||
|
{"browser_enabled": ["true"]},
|
||||||
|
["browser"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"computerEnabled=true",
|
||||||
|
{"computerEnabled": ["true"]},
|
||||||
|
["runtime"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"browserEnabled=true&enabled=true",
|
||||||
|
{"browserEnabled": ["true"], "enabled": ["true"]},
|
||||||
|
["browser", "runtime"],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_computer_use_update_route(monkeypatch) -> None:
|
async def test_computer_use_update_route(
|
||||||
|
monkeypatch,
|
||||||
|
query: str,
|
||||||
|
expected_query: dict[str, list[str]],
|
||||||
|
expected_sections: list[str],
|
||||||
|
) -> None:
|
||||||
captured: dict[str, object] = {}
|
captured: dict[str, object] = {}
|
||||||
|
|
||||||
def update(query):
|
def update(query):
|
||||||
@@ -150,7 +175,7 @@ async def test_computer_use_update_route(monkeypatch) -> None:
|
|||||||
|
|
||||||
monkeypatch.setattr("nanobot.webui.settings_routes.update_computer_use_settings", update)
|
monkeypatch.setattr("nanobot.webui.settings_routes.update_computer_use_settings", update)
|
||||||
request = SimpleNamespace(
|
request = SimpleNamespace(
|
||||||
path="/api/settings/computer-use/update?browser_enabled=true",
|
path=f"/api/settings/computer-use/update?{query}",
|
||||||
headers=Headers(),
|
headers=Headers(),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -162,4 +187,5 @@ async def test_computer_use_update_route(monkeypatch) -> None:
|
|||||||
|
|
||||||
assert response is not None
|
assert response is not None
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
assert captured["query"] == {"browser_enabled": ["true"]}
|
assert captured["query"] == expected_query
|
||||||
|
assert json.loads(response.body)["restart_required_sections"] == expected_sections
|
||||||
|
|||||||
Reference in New Issue
Block a user