diff --git a/nanobot/webui/settings_api.py b/nanobot/webui/settings_api.py index 1c77a293e..59947a52a 100644 --- a/nanobot/webui/settings_api.py +++ b/nanobot/webui/settings_api.py @@ -1261,6 +1261,11 @@ def settings_payload( "use_jina_reader": config.tools.web.fetch.use_jina_reader, }, }, + "computer_use": { + "browser_enabled": config.tools.browser.enable, + "enabled": config.tools.computer_use.enable, + "backend": config.tools.computer_use.backend, + }, "api": { "host": config.api.host, "port": config.api.port, @@ -2045,6 +2050,30 @@ def update_network_safety_settings(query: QueryParams) -> dict[str, Any]: return settings_payload(requires_restart=changed) +def update_computer_use_settings(query: QueryParams) -> dict[str, Any]: + raw_browser = _query_first_alias(query, "browser_enabled", "browserEnabled") + raw_computer = _query_first_alias(query, "enabled", "computerEnabled") + if raw_browser is None and raw_computer is None: + raise WebUISettingsError("browser_enabled or enabled is required") + + config = load_config() + changed = False + if raw_browser is not None: + browser_enabled = _parse_bool(raw_browser, "browser_enabled") + if config.tools.browser.enable != browser_enabled: + config.tools.browser.enable = browser_enabled + changed = True + if raw_computer is not None: + computer_enabled = _parse_bool(raw_computer, "enabled") + if config.tools.computer_use.enable != computer_enabled: + config.tools.computer_use.enable = computer_enabled + changed = True + + if changed: + save_config(config) + return settings_payload(requires_restart=changed) + + def update_web_search_settings(query: QueryParams) -> dict[str, Any]: provider_name = (_query_first(query, "provider") or "").strip().lower() provider_option = _WEB_SEARCH_PROVIDER_BY_NAME.get(provider_name) diff --git a/nanobot/webui/settings_routes.py b/nanobot/webui/settings_routes.py index 98cab4e7c..b2a8f8def 100644 --- a/nanobot/webui/settings_routes.py +++ b/nanobot/webui/settings_routes.py @@ -64,6 +64,7 @@ from nanobot.webui.settings_api import ( settings_usage_payload, update_agent_settings, update_api_settings, + update_computer_use_settings, update_image_generation_settings, update_model_call_order, update_model_configuration, @@ -174,6 +175,8 @@ class WebUISettingsRouter: return await self._handle_settings_provider_oauth(request, "logout") if path == "/api/settings/web-search/update": return self._handle_settings_web_search_update(request) + if path == "/api/settings/computer-use/update": + return self._handle_settings_computer_use_update(request) if path == "/api/settings/api-service": return self._handle_settings_api_service(request) if path == "/api/settings/api-service/start": @@ -506,6 +509,15 @@ class WebUISettingsRouter: return self._error_response(e.status, e.message) return self._json_response(self._with_restart_state(payload, section="browser")) + def _handle_settings_computer_use_update(self, request: WsRequest) -> Response: + if not self._authorized(request): + return self._unauthorized() + try: + payload = update_computer_use_settings(self._query(request)) + except WebUISettingsError as e: + return self._error_response(e.status, e.message) + return self._json_response(self._with_restart_state(payload, section="runtime")) + def _handle_settings_api_service(self, request: WsRequest) -> Response: if not self._authorized(request): return self._unauthorized() diff --git a/tests/webui/test_settings_api.py b/tests/webui/test_settings_api.py index 1151d902f..3112fec5e 100644 --- a/tests/webui/test_settings_api.py +++ b/tests/webui/test_settings_api.py @@ -29,6 +29,7 @@ from nanobot.webui.settings_api import ( settings_usage_payload, update_agent_settings, update_api_settings, + update_computer_use_settings, update_model_call_order, update_model_configuration, update_network_safety_settings, @@ -1007,6 +1008,27 @@ def test_settings_payload_includes_network_safety_fields( assert payload["advanced"]["ssrf_whitelist_count"] == 1 +def test_settings_payload_includes_computer_use_tools( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = tmp_path / "config.json" + config = Config() + config.tools.browser.enable = True + config.tools.computer_use.enable = True + config.tools.computer_use.backend = "browser" + save_config(config, config_path) + monkeypatch.setattr("nanobot.config.loader._current_config_path", config_path) + + payload = settings_payload() + + assert payload["computer_use"] == { + "browser_enabled": True, + "enabled": True, + "backend": "browser", + } + + def test_settings_payload_includes_exec_path_flags( tmp_path, monkeypatch: pytest.MonkeyPatch, @@ -1374,6 +1396,32 @@ def test_update_network_safety_settings_writes_local_service_flag( assert payload["requires_restart"] is True +def test_update_computer_use_settings_writes_only_requested_switches( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = tmp_path / "config.json" + save_config(Config(), config_path) + monkeypatch.setattr("nanobot.config.loader._current_config_path", config_path) + + browser_payload = update_computer_use_settings({"browser_enabled": ["true"]}) + saved = load_config(config_path) + assert saved.tools.browser.enable is True + assert saved.tools.computer_use.enable is False + assert browser_payload["requires_restart"] is True + + computer_payload = update_computer_use_settings({"computerEnabled": ["true"]}) + saved = load_config(config_path) + assert saved.tools.browser.enable is True + assert saved.tools.computer_use.enable is True + assert computer_payload["computer_use"]["enabled"] is True + + +def test_update_computer_use_settings_requires_a_switch() -> None: + with pytest.raises(WebUISettingsError, match="browser_enabled or enabled"): + update_computer_use_settings({}) + + def test_update_network_safety_settings_accepts_legacy_restricted_default_access( tmp_path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/webui/test_settings_routes.py b/tests/webui/test_settings_routes.py index 92dbec10f..a0573d26f 100644 --- a/tests/webui/test_settings_routes.py +++ b/tests/webui/test_settings_routes.py @@ -138,3 +138,28 @@ async def test_model_preset_mutation_routes( assert response.status_code == 200 assert json.loads(response.body)["routed"] == function_name assert captured["query"] == expected_query + + +@pytest.mark.asyncio +async def test_computer_use_update_route(monkeypatch) -> None: + captured: dict[str, object] = {} + + def update(query): + captured["query"] = query + return {"requires_restart": True} + + monkeypatch.setattr("nanobot.webui.settings_routes.update_computer_use_settings", update) + request = SimpleNamespace( + path="/api/settings/computer-use/update?browser_enabled=true", + headers=Headers(), + ) + + response = await _router().dispatch( + None, + request, + "/api/settings/computer-use/update", + ) + + assert response is not None + assert response.status_code == 200 + assert captured["query"] == {"browser_enabled": ["true"]} diff --git a/webui/src/components/settings/SettingsView.tsx b/webui/src/components/settings/SettingsView.tsx index ad1ee727e..ce3d46278 100644 --- a/webui/src/components/settings/SettingsView.tsx +++ b/webui/src/components/settings/SettingsView.tsx @@ -139,6 +139,7 @@ import { startApiService, stopApiService, updateAutomation, + updateComputerUseSettings, updateImageGenerationSettings, updateMcpServerTools, updateModelCallOrder, @@ -179,6 +180,7 @@ import type { AutomationUpdatePayload, CliAppInfo, CliAppsPayload, + ComputerUseSettingsUpdate, ImageGenerationSettingsUpdate, McpPresetInfo, McpPresetsPayload, @@ -762,6 +764,7 @@ export function SettingsView({ const [imageGenerationSaving, setImageGenerationSaving] = useState(false); const [transcriptionSaving, setTranscriptionSaving] = useState(false); const [networkSafetySaving, setNetworkSafetySaving] = useState(false); + const [computerUseSaving, setComputerUseSaving] = useState<"browser" | "computer" | null>(null); const [apiService, setApiService] = useState(null); const [apiServiceLoading, setApiServiceLoading] = useState(false); const [apiServiceAction, setApiServiceAction] = useState<"start" | "stop" | null>(null); @@ -1002,7 +1005,7 @@ export function SettingsView({ useEffect(() => { if ( !pageVisible - || !["channels", "models", "browser", "runtime"].includes(activeSection) + || !["channels", "models", "browser", "runtime", "advanced"].includes(activeSection) ) { return; } @@ -1559,6 +1562,34 @@ export function SettingsView({ } }; + const setComputerUseEnabled = async ( + target: "browser" | "computer", + enabled: boolean, + ) => { + if (!settings || computerUseSaving) return; + setComputerUseSaving(target); + try { + if (enabled && !(await installCapabilities(["computer-use"]))) return; + const update: ComputerUseSettingsUpdate = target === "browser" + ? { browserEnabled: enabled } + : { computerEnabled: enabled }; + const payload = await updateComputerUseSettings(token, update); + applyPayload(payload); + if (payload.requires_restart) { + setPendingRestartSections((prev) => ({ + ...prev, + [target === "browser" ? "browser" : "runtime"]: true, + })); + } + await maybeRestartHostEngine(payload); + setError(null); + } catch (err) { + setError((err as Error).message); + } finally { + setComputerUseSaving(null); + } + }; + const handleApiServiceAction = async ( action: "start" | "stop", values?: { host: string; port: number; timeout: number; apiKey?: string }, @@ -2238,6 +2269,11 @@ export function SettingsView({ olostepFeature={featureCatalog.find((feature) => feature.name === "olostep")} olostepInstalling={nanobotFeatureAction === "enable:olostep"} capabilityError={nanobotFeaturesError} + browserAutomationEnabled={settings.computer_use?.browser_enabled ?? false} + computerUseFeature={featureCatalog.find((feature) => feature.name === "computer-use")} + computerUseSaving={computerUseSaving === "browser"} + computerUseInstalling={nanobotFeatureAction === "enable:computer-use"} + onToggleBrowserAutomation={(enabled) => void setComputerUseEnabled("browser", enabled)} /> ); case "channels": @@ -2364,6 +2400,13 @@ export function SettingsView({ onRestart={restartViaSettingsSurface} isRestarting={isRestarting || hostEngineApplying} requiresRestartPending={pendingRestartSections.runtime} + computerControlEnabled={settings.computer_use?.enabled ?? false} + computerUseBackend={settings.computer_use?.backend ?? "desktop"} + computerUseFeature={featureCatalog.find((feature) => feature.name === "computer-use")} + computerUseSaving={computerUseSaving === "computer"} + computerUseInstalling={nanobotFeatureAction === "enable:computer-use"} + capabilityError={nanobotFeaturesError} + onToggleComputerControl={(enabled) => void setComputerUseEnabled("computer", enabled)} /> ); default: @@ -5255,6 +5298,11 @@ function WebSettings({ olostepFeature, olostepInstalling, capabilityError, + browserAutomationEnabled, + computerUseFeature, + computerUseSaving, + computerUseInstalling, + onToggleBrowserAutomation, }: { settings: SettingsPayload; form: WebSearchSettingsUpdate; @@ -5274,6 +5322,11 @@ function WebSettings({ olostepFeature?: NanobotFeatureInfo; olostepInstalling: boolean; capabilityError: string | null; + browserAutomationEnabled: boolean; + computerUseFeature?: NanobotFeatureInfo; + computerUseSaving: boolean; + computerUseInstalling: boolean; + onToggleBrowserAutomation: (enabled: boolean) => void; }) { const { t } = useTranslation(); const tx = (key: string, fallback: string) => t(key, { defaultValue: fallback }); @@ -5302,9 +5355,46 @@ function WebSettings({ : selectedProvider?.credential === "base_url" ? !baseUrl : false; + const computerUseInstalled = computerUseFeature?.installed ?? true; + const browserAutomationDescription = computerUseInstalling + ? tx("settings.help.computerUseInstalling", "Installing computer-use support...") + : computerUseInstalled + ? tx( + "settings.help.browserAutomation", + "Let nanobot navigate and act on web pages using structured page elements. Requires Playwright Chromium.", + ) + : tx( + "settings.help.computerUseInstall", + "Required Python support will be installed when you turn this on.", + ); return (
+
+ + {tx("settings.sections.browserAutomation", "Browser automation")} + + + + + + + {capabilityError ? ( +

{capabilityError}

+ ) : null} +
+
{tx("settings.sections.webSearch", "Web search")} {form.provider === "olostep" && olostepFeature && !olostepFeature.installed ? ( @@ -8765,6 +8855,13 @@ function AdvancedSettings({ onSave, onRestart, isRestarting, + computerControlEnabled, + computerUseBackend, + computerUseFeature, + computerUseSaving, + computerUseInstalling, + capabilityError, + onToggleComputerControl, }: { form: NetworkSafetySettingsUpdate; dirty: boolean; @@ -8775,11 +8872,60 @@ function AdvancedSettings({ onSave: () => void; onRestart?: () => void; isRestarting?: boolean; + computerControlEnabled: boolean; + computerUseBackend: "desktop" | "browser"; + computerUseFeature?: NanobotFeatureInfo; + computerUseSaving: boolean; + computerUseInstalling: boolean; + capabilityError: string | null; + onToggleComputerControl: (enabled: boolean) => void; }) { const { t } = useTranslation(); const tx = (key: string, fallback: string) => t(key, { defaultValue: fallback }); + const computerUseInstalled = computerUseFeature?.installed ?? true; + const computerControlDescription = computerUseInstalling + ? tx("settings.help.computerUseInstalling", "Installing computer-use support...") + : !computerUseInstalled + ? tx( + "settings.help.computerUseInstall", + "Required Python support will be installed when you turn this on.", + ) + : computerUseBackend === "browser" + ? tx( + "settings.help.computerControlBrowser", + "Pixel-based control currently targets an isolated browser, as configured in config.json.", + ) + : tx( + "settings.help.computerControl", + "Let nanobot see and control the computer running its engine. macOS requires Screen Recording and Accessibility access.", + ); return (
+
+ + {tx("settings.sections.computerControl", "Computer control")} + + + + + + + {capabilityError ? ( +

{capabilityError}

+ ) : null} +
+
{isNativeHostSurface diff --git a/webui/src/i18n/locales/en/common.json b/webui/src/i18n/locales/en/common.json index 72c8229df..af5b525ec 100644 --- a/webui/src/i18n/locales/en/common.json +++ b/webui/src/i18n/locales/en/common.json @@ -115,6 +115,8 @@ "imageDefaults": "Defaults", "webSearch": "Web search", "webBehavior": "Behavior", + "browserAutomation": "Browser automation", + "computerControl": "Computer control", "cliApps": "CLI apps", "mcp": "MCP services", "regional": "Regional", @@ -199,6 +201,8 @@ "maxResults": "Max results", "timeout": "Timeout", "jinaReader": "Jina reader", + "browserAutomation": "Browser automation", + "computerControl": "Computer control", "imageGeneration": "Image generation", "imageProvider": "Image provider", "imageProviderStatus": "Provider status", @@ -244,6 +248,11 @@ "maxResults": "Results returned by each web_search call.", "timeout": "Seconds before a search provider request times out.", "jinaReader": "Use Jina Reader for web_fetch when available.", + "browserAutomation": "Let nanobot navigate and act on web pages using structured page elements. Requires Playwright Chromium.", + "computerControl": "Let nanobot see and control the computer running its engine. macOS requires Screen Recording and Accessibility access.", + "computerControlBrowser": "Pixel-based control currently targets an isolated browser, as configured in config.json.", + "computerUseInstall": "Required Python support will be installed when you turn this on.", + "computerUseInstalling": "Installing computer-use support...", "imageGeneration": "Expose generate_image in chats when a configured image provider is available.", "imageProvider": "Choose the registry provider used by generate_image.", "imageProviderStatus": "Image generation reuses provider credentials from Providers.", diff --git a/webui/src/i18n/locales/es/common.json b/webui/src/i18n/locales/es/common.json index 6cb97d0cf..250ecb4b9 100644 --- a/webui/src/i18n/locales/es/common.json +++ b/webui/src/i18n/locales/es/common.json @@ -115,6 +115,8 @@ "imageDefaults": "Valores predeterminados", "webSearch": "Búsqueda web", "webBehavior": "Comportamiento", + "browserAutomation": "Automatización del navegador", + "computerControl": "Control del ordenador", "regional": "Configuración regional", "webuiSafety": "Seguridad de WebUI", "capabilities": "Capacidades", @@ -145,6 +147,8 @@ "maxResults": "Resultados máximos", "timeout": "Tiempo de espera", "jinaReader": "Lector Jina", + "browserAutomation": "Automatización del navegador", + "computerControl": "Control del ordenador", "imageGeneration": "Generación de imágenes", "imageProvider": "Proveedor de imágenes", "imageProviderStatus": "Estado del proveedor", @@ -188,6 +192,11 @@ "maxResults": "Resultados devueltos por cada llamada web_search.", "timeout": "Segundos antes de que una solicitud de búsqueda expire.", "jinaReader": "Usa Jina Reader para web_fetch cuando esté disponible.", + "browserAutomation": "Permite que nanobot navegue y actúe en páginas web mediante elementos estructurados. Requiere Chromium de Playwright.", + "computerControl": "Permite que nanobot vea y controle el ordenador que ejecuta su motor. macOS requiere permisos de grabación de pantalla y accesibilidad.", + "computerControlBrowser": "El control por píxeles apunta actualmente a un navegador aislado, según config.json.", + "computerUseInstall": "El soporte de Python necesario se instalará al activarlo.", + "computerUseInstalling": "Instalando componentes de control...", "imageGeneration": "Expone generate_image en chats cuando hay un proveedor de imagen configurado.", "imageProvider": "Elige el proveedor registrado usado por generate_image.", "imageProviderStatus": "La generación de imágenes reutiliza las credenciales de los proveedores.", diff --git a/webui/src/i18n/locales/fr/common.json b/webui/src/i18n/locales/fr/common.json index 266ff019a..161aeaf8d 100644 --- a/webui/src/i18n/locales/fr/common.json +++ b/webui/src/i18n/locales/fr/common.json @@ -115,6 +115,8 @@ "imageDefaults": "Valeurs par défaut", "webSearch": "Recherche web", "webBehavior": "Comportement", + "browserAutomation": "Automatisation du navigateur", + "computerControl": "Contrôle de l’ordinateur", "regional": "Paramètres régionaux", "webuiSafety": "Sécurité WebUI", "capabilities": "Capacités", @@ -145,6 +147,8 @@ "maxResults": "Résultats max.", "timeout": "Délai d’attente", "jinaReader": "Lecteur Jina", + "browserAutomation": "Automatisation du navigateur", + "computerControl": "Contrôle de l’ordinateur", "imageGeneration": "Génération d’images", "imageProvider": "Fournisseur d’images", "imageProviderStatus": "État du fournisseur", @@ -188,6 +192,11 @@ "maxResults": "Résultats renvoyés par chaque appel web_search.", "timeout": "Nombre de secondes avant l’expiration d’une requête de recherche.", "jinaReader": "Utilise Jina Reader pour web_fetch lorsque disponible.", + "browserAutomation": "Permet à nanobot de parcourir et manipuler les pages web à partir de leurs éléments structurés. Nécessite Chromium de Playwright.", + "computerControl": "Permet à nanobot de voir et contrôler l’ordinateur qui exécute son moteur. macOS exige les autorisations Enregistrement de l’écran et Accessibilité.", + "computerControlBrowser": "Le contrôle par pixels cible actuellement un navigateur isolé, conformément à config.json.", + "computerUseInstall": "Les composants Python requis seront installés lors de l’activation.", + "computerUseInstalling": "Installation des composants de contrôle...", "imageGeneration": "Expose generate_image dans les chats lorsqu’un fournisseur d’image configuré est disponible.", "imageProvider": "Choisissez le fournisseur inscrit utilisé par generate_image.", "imageProviderStatus": "La génération d’images réutilise les identifiants des fournisseurs.", diff --git a/webui/src/i18n/locales/id/common.json b/webui/src/i18n/locales/id/common.json index 975b5b8ed..d5665fbf5 100644 --- a/webui/src/i18n/locales/id/common.json +++ b/webui/src/i18n/locales/id/common.json @@ -115,6 +115,8 @@ "imageDefaults": "Bawaan", "webSearch": "Pencarian web", "webBehavior": "Perilaku", + "browserAutomation": "Otomatisasi browser", + "computerControl": "Kontrol komputer", "regional": "Regional", "webuiSafety": "Keamanan WebUI", "capabilities": "Kemampuan", @@ -145,6 +147,8 @@ "maxResults": "Hasil maksimum", "timeout": "Batas waktu", "jinaReader": "Pembaca Jina", + "browserAutomation": "Otomatisasi browser", + "computerControl": "Kontrol komputer", "imageGeneration": "Pembuatan gambar", "imageProvider": "Penyedia gambar", "imageProviderStatus": "Status penyedia", @@ -188,6 +192,11 @@ "maxResults": "Hasil yang dikembalikan oleh setiap panggilan web_search.", "timeout": "Detik sebelum permintaan penyedia pencarian mencapai batas waktu.", "jinaReader": "Gunakan Jina Reader untuk web_fetch jika tersedia.", + "browserAutomation": "Izinkan nanobot menjelajah dan bertindak pada halaman web melalui elemen terstruktur. Memerlukan Chromium dari Playwright.", + "computerControl": "Izinkan nanobot melihat dan mengontrol komputer yang menjalankan mesinnya. macOS memerlukan izin Perekaman Layar dan Aksesibilitas.", + "computerControlBrowser": "Kontrol berbasis piksel saat ini menargetkan browser terisolasi sesuai config.json.", + "computerUseInstall": "Dukungan Python yang diperlukan akan dipasang saat diaktifkan.", + "computerUseInstalling": "Memasang komponen kontrol komputer...", "imageGeneration": "Tampilkan generate_image di chat saat penyedia gambar yang dikonfigurasi tersedia.", "imageProvider": "Pilih penyedia registry yang digunakan oleh generate_image.", "imageProviderStatus": "Pembuatan gambar menggunakan kembali kredensial penyedia dari bagian Penyedia.", diff --git a/webui/src/i18n/locales/ja/common.json b/webui/src/i18n/locales/ja/common.json index 968ac0993..399ae1bb1 100644 --- a/webui/src/i18n/locales/ja/common.json +++ b/webui/src/i18n/locales/ja/common.json @@ -115,6 +115,8 @@ "imageDefaults": "既定値", "webSearch": "ウェブ検索", "webBehavior": "動作", + "browserAutomation": "ブラウザ自動操作", + "computerControl": "コンピュータ操作", "regional": "地域", "webuiSafety": "WebUI の安全性", "capabilities": "機能", @@ -145,6 +147,8 @@ "maxResults": "最大結果数", "timeout": "タイムアウト", "jinaReader": "Jina リーダー", + "browserAutomation": "ブラウザ自動操作", + "computerControl": "コンピュータ操作", "imageGeneration": "画像生成", "imageProvider": "画像プロバイダー", "imageProviderStatus": "プロバイダー状態", @@ -188,6 +192,11 @@ "maxResults": "各 web_search 呼び出しで返す結果数です。", "timeout": "検索プロバイダーのリクエストがタイムアウトするまでの秒数です。", "jinaReader": "利用可能な場合、web_fetch に Jina Reader を使います。", + "browserAutomation": "構造化されたページ要素を使って、nanobot がウェブページを閲覧・操作できるようにします。Playwright Chromium が必要です。", + "computerControl": "nanobot がエンジンを実行しているコンピュータを表示・操作できるようにします。macOS では画面収録とアクセシビリティの許可が必要です。", + "computerControlBrowser": "ピクセル操作は現在、config.json の設定に従って分離ブラウザを対象としています。", + "computerUseInstall": "有効にすると必要な Python コンポーネントがインストールされます。", + "computerUseInstalling": "コンピュータ操作コンポーネントをインストール中...", "imageGeneration": "画像プロバイダーが設定済みのとき、チャットで generate_image を有効にします。", "imageProvider": "generate_image で使用する登録済みプロバイダーを選択します。", "imageProviderStatus": "画像生成はプロバイダー設定の認証情報を再利用します。", diff --git a/webui/src/i18n/locales/ko/common.json b/webui/src/i18n/locales/ko/common.json index 8b6ec667f..46d933812 100644 --- a/webui/src/i18n/locales/ko/common.json +++ b/webui/src/i18n/locales/ko/common.json @@ -115,6 +115,8 @@ "imageDefaults": "기본값", "webSearch": "웹 검색", "webBehavior": "동작", + "browserAutomation": "브라우저 자동화", + "computerControl": "컴퓨터 제어", "regional": "지역", "webuiSafety": "WebUI 보안", "capabilities": "기능", @@ -145,6 +147,8 @@ "maxResults": "최대 결과 수", "timeout": "타임아웃", "jinaReader": "Jina 리더", + "browserAutomation": "브라우저 자동화", + "computerControl": "컴퓨터 제어", "imageGeneration": "이미지 생성", "imageProvider": "이미지 제공자", "imageProviderStatus": "제공자 상태", @@ -188,6 +192,11 @@ "maxResults": "각 web_search 호출에서 반환되는 결과 수입니다.", "timeout": "검색 제공자 요청이 타임아웃되기 전의 초입니다.", "jinaReader": "가능할 때 web_fetch에 Jina Reader를 사용합니다.", + "browserAutomation": "nanobot이 구조화된 페이지 요소를 사용해 웹페이지를 탐색하고 조작할 수 있게 합니다. Playwright Chromium이 필요합니다.", + "computerControl": "nanobot이 엔진을 실행하는 컴퓨터를 보고 제어할 수 있게 합니다. macOS에서는 화면 기록 및 손쉬운 사용 권한이 필요합니다.", + "computerControlBrowser": "픽셀 기반 제어는 현재 config.json 설정에 따라 격리된 브라우저를 대상으로 합니다.", + "computerUseInstall": "켜면 필요한 Python 구성 요소가 설치됩니다.", + "computerUseInstalling": "컴퓨터 제어 구성 요소 설치 중...", "imageGeneration": "구성된 이미지 제공자가 있을 때 채팅에서 generate_image를 노출합니다.", "imageProvider": "generate_image에 사용할 등록 제공자를 선택합니다.", "imageProviderStatus": "이미지 생성은 제공자 자격 증명을 재사용합니다.", diff --git a/webui/src/i18n/locales/pt-BR/common.json b/webui/src/i18n/locales/pt-BR/common.json index 9fbe9cbb8..9ad9f3693 100644 --- a/webui/src/i18n/locales/pt-BR/common.json +++ b/webui/src/i18n/locales/pt-BR/common.json @@ -115,6 +115,8 @@ "imageDefaults": "Padrões", "webSearch": "Busca na web", "webBehavior": "Comportamento", + "browserAutomation": "Automação do navegador", + "computerControl": "Controle do computador", "cliApps": "Aplicativos CLI", "mcp": "Servidores MCP", "regional": "Regional", @@ -199,6 +201,8 @@ "maxResults": "Máx. de resultados", "timeout": "Tempo limite", "jinaReader": "Leitor Jina", + "browserAutomation": "Automação do navegador", + "computerControl": "Controle do computador", "imageGeneration": "Geração de imagens", "imageProvider": "Provedor de imagem", "imageProviderStatus": "Status do provedor", @@ -244,6 +248,11 @@ "maxResults": "Resultados retornados por cada chamada de web_search.", "timeout": "Segundos antes de uma requisição de busca expirar.", "jinaReader": "Usa o Jina Reader para web_fetch quando disponível.", + "browserAutomation": "Permite que o nanobot navegue e interaja com páginas usando elementos estruturados. Requer o Chromium do Playwright.", + "computerControl": "Permite que o nanobot veja e controle o computador que executa o mecanismo. No macOS, requer acesso à Gravação de Tela e Acessibilidade.", + "computerControlBrowser": "O controle por pixels está direcionado a um navegador isolado, conforme o config.json.", + "computerUseInstall": "O suporte Python necessário será instalado ao ativar.", + "computerUseInstalling": "Instalando componentes de controle...", "imageGeneration": "Expõe generate_image nas conversas quando há um provedor de imagem configurado.", "imageProvider": "Escolha o provedor do registro usado por generate_image.", "imageProviderStatus": "A geração de imagens reaproveita as credenciais dos provedores.", diff --git a/webui/src/i18n/locales/vi/common.json b/webui/src/i18n/locales/vi/common.json index e9a79be0d..bfef5b965 100644 --- a/webui/src/i18n/locales/vi/common.json +++ b/webui/src/i18n/locales/vi/common.json @@ -115,6 +115,8 @@ "imageDefaults": "Mặc định", "webSearch": "Tìm kiếm web", "webBehavior": "Hành vi", + "browserAutomation": "Tự động hóa trình duyệt", + "computerControl": "Điều khiển máy tính", "regional": "Khu vực", "webuiSafety": "An toàn WebUI", "capabilities": "Khả năng", @@ -145,6 +147,8 @@ "maxResults": "Kết quả tối đa", "timeout": "Thời gian chờ", "jinaReader": "Trình đọc Jina", + "browserAutomation": "Tự động hóa trình duyệt", + "computerControl": "Điều khiển máy tính", "imageGeneration": "Tạo hình ảnh", "imageProvider": "Nhà cung cấp hình ảnh", "imageProviderStatus": "Trạng thái nhà cung cấp", @@ -188,6 +192,11 @@ "maxResults": "Số kết quả được trả về sau mỗi lần gọi web_search.", "timeout": "Số giây trước khi yêu cầu của nhà cung cấp tìm kiếm hết thời gian.", "jinaReader": "Dùng Jina Reader cho web_fetch khi có thể.", + "browserAutomation": "Cho phép nanobot duyệt và thao tác trên trang web bằng các phần tử có cấu trúc. Cần Playwright Chromium.", + "computerControl": "Cho phép nanobot xem và điều khiển máy tính đang chạy engine. macOS yêu cầu quyền Ghi màn hình và Trợ năng.", + "computerControlBrowser": "Điều khiển theo điểm ảnh hiện nhắm tới trình duyệt cách ly theo config.json.", + "computerUseInstall": "Hỗ trợ Python cần thiết sẽ được cài đặt khi bật.", + "computerUseInstalling": "Đang cài đặt thành phần điều khiển...", "imageGeneration": "Hiển thị generate_image trong chat khi đã cấu hình nhà cung cấp hình ảnh.", "imageProvider": "Chọn nhà cung cấp registry được generate_image sử dụng.", "imageProviderStatus": "Tạo ảnh dùng lại thông tin xác thực từ mục Nhà cung cấp.", diff --git a/webui/src/i18n/locales/zh-CN/common.json b/webui/src/i18n/locales/zh-CN/common.json index 224c07ba5..e7feeef11 100644 --- a/webui/src/i18n/locales/zh-CN/common.json +++ b/webui/src/i18n/locales/zh-CN/common.json @@ -115,6 +115,8 @@ "imageDefaults": "默认值", "webSearch": "网络搜索", "webBehavior": "网络行为", + "browserAutomation": "浏览器自动化", + "computerControl": "电脑控制", "cliApps": "CLI 应用", "mcp": "MCP 服务", "regional": "区域", @@ -199,6 +201,8 @@ "maxResults": "最大结果数", "timeout": "超时", "jinaReader": "Jina 阅读器", + "browserAutomation": "浏览器自动化", + "computerControl": "电脑控制", "imageGeneration": "图片生成", "imageProvider": "图片提供商", "imageProviderStatus": "提供商状态", @@ -244,6 +248,11 @@ "maxResults": "每次 web_search 调用返回的结果数。", "timeout": "搜索提供商请求超时前等待的秒数。", "jinaReader": "可用时为 web_fetch 使用 Jina Reader。", + "browserAutomation": "允许 nanobot 通过结构化页面元素浏览网页并执行操作,需要 Playwright Chromium。", + "computerControl": "允许 nanobot 查看并控制运行引擎的电脑。macOS 需要授予“屏幕录制”和“辅助功能”权限。", + "computerControlBrowser": "像素控制当前按 config.json 配置作用于隔离浏览器。", + "computerUseInstall": "开启时会自动安装所需 Python 组件。", + "computerUseInstalling": "正在安装电脑操作组件...", "imageGeneration": "配置图片提供商后,即可在聊天中使用 generate_image。", "imageProvider": "选择 generate_image 使用的注册提供商。", "imageProviderStatus": "图片生成会复用「提供商」里的凭据。", diff --git a/webui/src/i18n/locales/zh-TW/common.json b/webui/src/i18n/locales/zh-TW/common.json index a46be8969..ce2273ecf 100644 --- a/webui/src/i18n/locales/zh-TW/common.json +++ b/webui/src/i18n/locales/zh-TW/common.json @@ -115,6 +115,8 @@ "imageDefaults": "預設值", "webSearch": "網路搜尋", "webBehavior": "網路行為", + "browserAutomation": "瀏覽器自動化", + "computerControl": "電腦控制", "regional": "區域", "webuiSafety": "WebUI 安全", "capabilities": "能力", @@ -145,6 +147,8 @@ "maxResults": "最大結果數", "timeout": "逾時", "jinaReader": "Jina 閱讀器", + "browserAutomation": "瀏覽器自動化", + "computerControl": "電腦控制", "imageGeneration": "圖片生成", "imageProvider": "圖片供應商", "imageProviderStatus": "供應商狀態", @@ -188,6 +192,11 @@ "maxResults": "每次呼叫 web_search 所回傳的結果數。", "timeout": "搜尋供應商請求逾時前的秒數。", "jinaReader": "若可用,則讓 web_fetch 使用 Jina Reader。", + "browserAutomation": "允許 nanobot 透過結構化頁面元素瀏覽網頁並執行操作,需要 Playwright Chromium。", + "computerControl": "允許 nanobot 查看並控制執行引擎的電腦。macOS 需要授予螢幕錄製與輔助使用權限。", + "computerControlBrowser": "像素控制目前依 config.json 設定作用於隔離瀏覽器。", + "computerUseInstall": "開啟時會自動安裝所需 Python 元件。", + "computerUseInstalling": "正在安裝電腦操作元件...", "imageGeneration": "設定圖片供應商後,即可在聊天中使用 generate_image。", "imageProvider": "選擇 generate_image 使用的註冊供應商。", "imageProviderStatus": "圖片生成功能會沿用 [供應商] 中的憑證。", diff --git a/webui/src/lib/api.ts b/webui/src/lib/api.ts index f1e416c05..d394aeaf7 100644 --- a/webui/src/lib/api.ts +++ b/webui/src/lib/api.ts @@ -7,6 +7,7 @@ import type { ChannelValidationPayload, ChatSummary, CliAppsPayload, + ComputerUseSettingsUpdate, FilePreviewPayload, ImageGenerationSettingsUpdate, McpPresetsPayload, @@ -1077,6 +1078,24 @@ export async function updateNetworkSafetySettings( ); } +export async function updateComputerUseSettings( + token: string, + update: ComputerUseSettingsUpdate, + base: string = "", +): Promise { + const query = new URLSearchParams(); + if (update.browserEnabled !== undefined) { + query.set("browser_enabled", String(update.browserEnabled)); + } + if (update.computerEnabled !== undefined) { + query.set("enabled", String(update.computerEnabled)); + } + return request( + `${base}/api/settings/computer-use/update?${query}`, + token, + ); +} + export async function updateImageGenerationSettings( token: string, update: ImageGenerationSettingsUpdate, diff --git a/webui/src/lib/types.ts b/webui/src/lib/types.ts index b40026832..fba91dfa3 100644 --- a/webui/src/lib/types.ts +++ b/webui/src/lib/types.ts @@ -579,6 +579,11 @@ export interface SettingsPayload { use_jina_reader: boolean; }; }; + computer_use?: { + browser_enabled: boolean; + enabled: boolean; + backend: "desktop" | "browser"; + }; api?: { host: string; port: number; @@ -1102,6 +1107,11 @@ export interface NetworkSafetySettingsUpdate { webuiDefaultAccessMode: WebuiDefaultAccessMode; } +export interface ComputerUseSettingsUpdate { + browserEnabled?: boolean; + computerEnabled?: boolean; +} + export interface ImageGenerationSettingsUpdate { enabled: boolean; provider: string; diff --git a/webui/src/tests/api.test.ts b/webui/src/tests/api.test.ts index 634835031..66e179628 100644 --- a/webui/src/tests/api.test.ts +++ b/webui/src/tests/api.test.ts @@ -46,6 +46,7 @@ import { pollChannelConnect, startChannelConnect, updateAutomation, + updateComputerUseSettings, updateSidebarState, updateImageGenerationSettings, updateModelCallOrder, @@ -836,6 +837,20 @@ describe("webui API helpers", () => { ); }); + it("updates computer-use capability switches", async () => { + await updateComputerUseSettings("tok", { browserEnabled: true }); + expect(fetch).toHaveBeenCalledWith( + "/api/settings/computer-use/update?browser_enabled=true", + expect.objectContaining({ headers: { Authorization: "Bearer tok" } }), + ); + + await updateComputerUseSettings("tok", { computerEnabled: false }); + expect(fetch).toHaveBeenCalledWith( + "/api/settings/computer-use/update?enabled=false", + expect.objectContaining({ headers: { Authorization: "Bearer tok" } }), + ); + }); + it("manages the API service capability", async () => { await fetchApiService("tok"); expect(fetch).toHaveBeenCalledWith( diff --git a/webui/src/tests/i18n.test.tsx b/webui/src/tests/i18n.test.tsx index 2c0bcba47..a1fa4b5d1 100644 --- a/webui/src/tests/i18n.test.tsx +++ b/webui/src/tests/i18n.test.tsx @@ -69,6 +69,8 @@ const LOCALIZED_SETTINGS_COPY_KEYS = [ "settings.sections.localPreferences", "settings.sections.webSearch", "settings.sections.webBehavior", + "settings.sections.browserAutomation", + "settings.sections.computerControl", "settings.sections.webuiSafety", "settings.sections.capabilities", "settings.sections.apps", @@ -149,6 +151,8 @@ const LOCALIZED_SETTINGS_COPY_KEYS = [ "settings.rows.fileEditDisplay", "settings.rows.codeWrap", "settings.rows.brandLogos", + "settings.rows.browserAutomation", + "settings.rows.computerControl", "settings.rows.currentModel", "settings.rows.localServiceAccess", "settings.rows.webuiDefaultAccess", @@ -160,6 +164,11 @@ const LOCALIZED_SETTINGS_COPY_KEYS = [ "settings.help.fileEditDisplay", "settings.help.codeWrap", "settings.help.brandLogos", + "settings.help.browserAutomation", + "settings.help.computerControl", + "settings.help.computerControlBrowser", + "settings.help.computerUseInstall", + "settings.help.computerUseInstalling", "settings.help.currentModel", "settings.help.localServiceAccess", "settings.help.webuiDefaultAccess", diff --git a/webui/src/tests/settings-view.test.tsx b/webui/src/tests/settings-view.test.tsx index 96e7fdbe8..b68d6156d 100644 --- a/webui/src/tests/settings-view.test.tsx +++ b/webui/src/tests/settings-view.test.tsx @@ -64,6 +64,11 @@ function settingsPayload(): SettingsPayload { search: { max_results: 5, timeout: 30 }, fetch: { use_jina_reader: true }, }, + computer_use: { + browser_enabled: false, + enabled: false, + backend: "desktop", + }, api: { host: "127.0.0.1", port: 8900, @@ -4202,6 +4207,98 @@ describe("SettingsView Apps catalog", () => { }); }); + it("enables browser automation from Web settings", async () => { + const payload = settingsPayload(); + const updatedPayload: SettingsPayload = { + ...payload, + computer_use: { ...payload.computer_use!, browser_enabled: true }, + requires_restart: true, + restart_required_sections: ["runtime"], + }; + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url === "/api/settings") return jsonResponse(payload); + if (url === "/api/settings/nanobot-features") { + return jsonResponse({ + features: [{ + name: "computer-use", + display_name: "Computer Use", + type: "feature", + enabled: true, + installed: true, + ready: true, + status: "enabled", + install_supported: true, + requires_restart: true, + }], + enabled_count: 1, + }); + } + if (url === "/api/settings/computer-use/update?browser_enabled=true") { + return jsonResponse(updatedPayload); + } + return { ok: false, status: 404, json: async () => ({}) } as Response; + }); + vi.stubGlobal("fetch", fetchMock); + + renderSettingsView({ initialSection: "browser" }); + + fireEvent.click(await screen.findByRole("switch", { name: "Browser automation" })); + + await waitFor(() => + expect(fetchMock).toHaveBeenCalledWith( + "/api/settings/computer-use/update?browser_enabled=true", + expect.objectContaining({ headers: { Authorization: "Bearer tok" } }), + ), + ); + }); + + it("enables computer control from Security settings", async () => { + const payload = settingsPayload(); + const updatedPayload: SettingsPayload = { + ...payload, + computer_use: { ...payload.computer_use!, enabled: true }, + requires_restart: true, + restart_required_sections: ["runtime"], + }; + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url === "/api/settings") return jsonResponse(payload); + if (url === "/api/settings/nanobot-features") { + return jsonResponse({ + features: [{ + name: "computer-use", + display_name: "Computer Use", + type: "feature", + enabled: true, + installed: true, + ready: true, + status: "enabled", + install_supported: true, + requires_restart: true, + }], + enabled_count: 1, + }); + } + if (url === "/api/settings/computer-use/update?enabled=true") { + return jsonResponse(updatedPayload); + } + return { ok: false, status: 404, json: async () => ({}) } as Response; + }); + vi.stubGlobal("fetch", fetchMock); + + renderSettingsView({ initialSection: "advanced" }); + + fireEvent.click(await screen.findByRole("switch", { name: "Computer control" })); + + await waitFor(() => + expect(fetchMock).toHaveBeenCalledWith( + "/api/settings/computer-use/update?enabled=true", + expect.objectContaining({ headers: { Authorization: "Bearer tok" } }), + ), + ); + }); + it("saves network safety without exposing technical SSRF copy", async () => { const payload = settingsPayload(); const fetchMock = vi.fn(async (input: RequestInfo | URL) => {