From 12029f88127ff2692cb5b5f72237139d2c7c9ad7 Mon Sep 17 00:00:00 2001 From: Xubin Ren <52506698+Re-bin@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:33:22 +0800 Subject: [PATCH] feat(runtime): add user-controlled turn recovery --- nanobot/agent/loop.py | 235 ++--- nanobot/agent/runner.py | 18 + nanobot/bus/outbound_events.py | 9 + nanobot/channels/manager.py | 11 + nanobot/channels/websocket/runtime.py | 30 + .../websocket/tests/test_websocket_channel.py | 34 + .../tests/test_websocket_http_routes.py | 26 + nanobot/cli/gateway_runtime.py | 22 + nanobot/gateway/runtime.py | 56 +- nanobot/session/manager.py | 137 ++- nanobot/session/recovery.py | 911 ++++++++++++++++++ nanobot/session/webui_turns.py | 4 + nanobot/webui/gateway_services.py | 2 + nanobot/webui/session_list_index.py | 7 +- nanobot/webui/transcript.py | 13 + nanobot/webui/ws_http.py | 39 + tests/agent/test_loop_save_turn.py | 58 ++ tests/agent/test_runner_injections.py | 100 +- tests/agent/test_stop_preserves_context.py | 22 + tests/channels/test_channel_plugins.py | 21 + tests/cli/test_commands.py | 24 +- tests/gateway/test_runtime.py | 43 + tests/session/test_recovery.py | 732 ++++++++++++++ tests/session/test_session_store.py | 91 ++ tests/utils/test_webui_transcript.py | 18 + tests/webui/test_gateway_webui_smoke.py | 84 ++ tests/webui/test_session_list_index.py | 25 + tui/src/app.test.ts | 120 ++- tui/src/app.ts | 125 ++- tui/src/protocol.test.ts | 56 ++ tui/src/protocol.ts | 133 ++- tui/src/recovery-notice.ts | 155 +++ tui/src/session-menu.test.ts | 7 +- tui/src/session-menu.ts | 8 +- webui/src/App.tsx | 10 + webui/src/components/ChatList.tsx | 42 +- webui/src/components/Sidebar.tsx | 2 + .../settings/overview/OverviewSettings.tsx | 31 + .../src/components/thread/RecoveryNotice.tsx | 108 +++ webui/src/components/thread/ThreadShell.tsx | 22 +- webui/src/hooks/useNanobotStream.ts | 102 ++ webui/src/i18n/locales/en/common.json | 17 +- webui/src/i18n/locales/es/common.json | 17 +- webui/src/i18n/locales/fr/common.json | 17 +- webui/src/i18n/locales/id/common.json | 17 +- webui/src/i18n/locales/ja/common.json | 17 +- webui/src/i18n/locales/ko/common.json | 17 +- webui/src/i18n/locales/pt-BR/common.json | 17 +- webui/src/i18n/locales/vi/common.json | 17 +- webui/src/i18n/locales/zh-CN/common.json | 17 +- webui/src/i18n/locales/zh-TW/common.json | 17 +- webui/src/lib/api.ts | 3 + webui/src/lib/local-preferences.ts | 3 + webui/src/lib/types.ts | 17 + webui/src/tests/chat-list.test.tsx | 38 + webui/src/tests/i18n.test.tsx | 13 + webui/src/tests/local-preferences.test.ts | 19 + webui/src/tests/recovery-notice.test.tsx | 114 +++ webui/src/tests/thread-shell.test.tsx | 28 + webui/src/tests/useNanobotStream.test.tsx | 98 ++ 60 files changed, 4027 insertions(+), 169 deletions(-) create mode 100644 nanobot/session/recovery.py create mode 100644 tests/session/test_recovery.py create mode 100644 tui/src/recovery-notice.ts create mode 100644 webui/src/components/thread/RecoveryNotice.tsx create mode 100644 webui/src/tests/local-preferences.test.ts create mode 100644 webui/src/tests/recovery-notice.test.tsx diff --git a/nanobot/agent/loop.py b/nanobot/agent/loop.py index 184cb4a29..3782dae4b 100644 --- a/nanobot/agent/loop.py +++ b/nanobot/agent/loop.py @@ -79,6 +79,15 @@ from nanobot.session.model_selection import ( SESSION_MODEL_PRESET_METADATA_KEY, model_preset_from_metadata, ) +from nanobot.session.recovery import ( + PENDING_FOLLOWUP_ID_KEY, + RECOVERY_INBOUND_METADATA_KEY, + RecoveryAdmission, + acknowledge_pending_followups, + record_pending_followup, + restore_pending_interruption, + restore_runtime_checkpoint, +) from nanobot.session.summary import SessionSummary from nanobot.triggers.local_turns import LocalTriggerTurnCoordinator from nanobot.utils.cancellation import task_is_cancelling @@ -291,12 +300,14 @@ class AgentLoop: restart_mode: str = "auto", local_trigger_store: LocalTriggerStore | None = None, idle_compact_check_interval_seconds: int = 0, + recovery_admission: RecoveryAdmission | None = None, ): from nanobot.config.schema import ToolsConfig _tc = tools_config or ToolsConfig() defaults = AgentDefaults() self.bus = bus + self._recovery_admission = recovery_admission if turn_delivery_factory is not None: if turn_delivery_factory.bus is not bus: raise ValueError("turn delivery factory must use the agent message bus") @@ -409,6 +420,7 @@ class AgentLoop: # When a session has an active task, new messages for that session # are routed here instead of creating a new task. self._pending_queues: dict[str, asyncio.Queue[InboundMessage]] = {} + self._preserve_inflight_turns_on_shutdown = False self._deferred_automation_turns: dict[str, list[InboundMessage]] = {} self._cron_turns = CronTurnCoordinator( publish_inbound=self.bus.publish_inbound, @@ -726,6 +738,9 @@ class AgentLoop: extra[RUNTIME_CONTEXT_HISTORY_META] = runtime_context_meta session.add_message("user", text, **extra) self._mark_pending_user_turn(session) + followup_id = msg.metadata.get(PENDING_FOLLOWUP_ID_KEY) + if isinstance(followup_id, str) and followup_id: + acknowledge_pending_followups(session, [followup_id]) self.sessions.save(session) return True return False @@ -1061,6 +1076,9 @@ class AgentLoop: row["subagent_task_id"] = task_id row[HIDDEN_HISTORY_META] = subagent_marker row["injected_event"] = "subagent_result" + followup_id = metadata.get(PENDING_FOLLOWUP_ID_KEY) + if isinstance(followup_id, str) and followup_id: + row[PENDING_FOLLOWUP_ID_KEY] = followup_id return row items: list[dict[str, Any]] = [] @@ -1285,6 +1303,17 @@ class AgentLoop: break if deferred: continue + # A newer WebUI message must supersede an explicit recovery + # before it is injected into that recovery's pending queue. + # Without this admission point, a recovered turn could finish + # first and only then observe the user's newer request. + if ( + effective_key in self._pending_queues + and msg.channel == "websocket" + and self._recovery_admission is not None + and not await self._recovery_admission.admit(msg) + ): + continue # If this session already has an active pending queue (i.e. a task # is processing this session), route the message there for mid-turn # injection instead of creating a competing task. @@ -1303,6 +1332,17 @@ class AgentLoop: msg, session_key_override=effective_key, ) + session = self.sessions.get_or_create(effective_key) + followup_id = record_pending_followup(session, pending_msg) + if followup_id is not None: + pending_msg = dataclasses.replace( + pending_msg, + metadata={ + **pending_msg.metadata, + PENDING_FOLLOWUP_ID_KEY: followup_id, + }, + ) + self.sessions.save(session) try: self._pending_queues[effective_key].put_nowait(pending_msg) except asyncio.QueueFull: @@ -1310,6 +1350,7 @@ class AgentLoop: "Pending queue full for session {}, falling back to queued task", effective_key, ) + msg = pending_msg else: logger.info( "Routed follow-up message to pending queue for session {}", @@ -1319,17 +1360,45 @@ class AgentLoop: # Compute the effective session key before dispatching # This ensures /stop command can find tasks correctly when unified session is enabled task = asyncio.create_task(self._dispatch(msg)) - active_tasks = self._active_tasks.setdefault(effective_key, set()) + active_tasks: set[asyncio.Task[Any]] = self._active_tasks.setdefault( + effective_key, + set(), + ) active_tasks.add(task) task.add_done_callback(active_tasks.discard) finally: await self.aclose() + def preserve_inflight_turns_on_shutdown(self) -> None: + """Keep durable checkpoints when the owning gateway is restarting. + + Normal cancellation intentionally materializes partial output so a + stopped gateway leaves a readable conversation. A managed restart is + different: RecoveryCoordinator needs the checkpoint intact to safely + offer the unfinished turn for explicit continuation after restart. + """ + self._preserve_inflight_turns_on_shutdown = True + async def _dispatch(self, msg: InboundMessage) -> None: """Process a message: per-session serial, cross-session concurrent.""" session_key = self._effective_session_key(msg) if session_key != msg.session_key: msg = dataclasses.replace(msg, session_key_override=session_key) + recovery_task_registered = False + recovery_admission = self._recovery_admission + current_task: asyncio.Task[Any] | None = None + if recovery_admission is not None: + recovery_id = msg.metadata.get(RECOVERY_INBOUND_METADATA_KEY) + if isinstance(recovery_id, str) and recovery_id: + current_task = asyncio.current_task() + if current_task is not None: + recovery_admission.register_recovery_task(session_key, current_task) + recovery_task_registered = True + if not await recovery_admission.admit(msg): + logger.info("Skipped stale recovery for session {}", session_key) + if recovery_task_registered and current_task is not None: + recovery_admission.unregister_recovery_task(session_key, current_task) + return lock = self._get_session_lock(session_key) gate = self._concurrency_gate or nullcontext() @@ -1380,7 +1449,10 @@ class AgentLoop: # _emit_checkpoint during tool execution; materializing # it into session history now makes it visible in the # next conversation turn. - if session_key in self._discarding_sessions: + if ( + session_key in self._discarding_sessions + or self._preserve_inflight_turns_on_shutdown + ): raise try: key = self._effective_session_key(msg) @@ -1437,6 +1509,12 @@ class AgentLoop: await delivery.idle() await self._publish_next_deferred_automation_turn(session_key) finally: + if ( + recovery_task_registered + and current_task is not None + and recovery_admission is not None + ): + recovery_admission.unregister_recovery_task(session_key, current_task) if pending is None: await delivery.idle() await self._publish_next_deferred_automation_turn(session_key) @@ -1738,7 +1816,10 @@ class AgentLoop: if self._restore_runtime_checkpoint(session): self.sessions.save(session) - if self._restore_pending_user_turn(session): + if ( + RECOVERY_INBOUND_METADATA_KEY not in msg.metadata + and restore_pending_interruption(session) + ): self.sessions.save(session) async def _compact_session(self, ctx: TurnContext) -> None: @@ -2093,8 +2174,21 @@ class AgentLoop: if m.get("role") == "tool" and m.get("tool_call_id") } last_assistant_idx: int | None = None + saved_followup_ids: set[str] = set() for m in messages[skip:]: entry = dict(m) + followup_id_value = cast(object, entry.pop(PENDING_FOLLOWUP_ID_KEY, None)) + followup_ids = ( + [followup_id_value] + if isinstance(followup_id_value, str) + else [ + followup_id + for followup_id in cast(list[object], followup_id_value) + if isinstance(followup_id, str) + ] + if isinstance(followup_id_value, list) + else [] + ) internal_meta = cast(object, entry.pop("_meta", None)) runtime_context_meta = ( cast(dict[str, Any], internal_meta).get( @@ -2147,6 +2241,8 @@ class AgentLoop: entry[RUNTIME_CONTEXT_HISTORY_META] = runtime_context_meta entry.setdefault("timestamp", datetime.now().isoformat()) session.messages.append(entry) + if role == "user": + saved_followup_ids.update(followup_id for followup_id in followup_ids if followup_id) if role == "assistant": last_assistant_idx = len(session.messages) - 1 declared_tool_call_ids.update( @@ -2161,6 +2257,8 @@ class AgentLoop: ) if turn_latency_ms is not None and last_assistant_idx is not None: session.messages[last_assistant_idx]["latency_ms"] = int(turn_latency_ms) + if saved_followup_ids: + acknowledge_pending_followups(session, saved_followup_ids) session.updated_at = datetime.now() def _persist_subagent_followup(self, session: Session, msg: InboundMessage) -> bool: @@ -2195,7 +2293,7 @@ class AgentLoop: def _set_runtime_checkpoint(self, session: Session, payload: dict[str, Any]) -> None: """Persist the latest in-flight turn state into session metadata.""" session.metadata[self._RUNTIME_CHECKPOINT_KEY] = payload - self.sessions.save(session) + self.sessions.save_runtime_checkpoint(session) def _mark_pending_user_turn(self, session: Session) -> None: session.metadata[self._PENDING_USER_TURN_KEY] = True @@ -2207,136 +2305,9 @@ class AgentLoop: if self._RUNTIME_CHECKPOINT_KEY in session.metadata: session.metadata.pop(self._RUNTIME_CHECKPOINT_KEY, None) - @staticmethod - def _checkpoint_message_key(message: dict[str, Any]) -> tuple[Any, ...]: - return ( - message.get("role"), - message.get("content"), - message.get("tool_call_id"), - message.get("name"), - message.get("tool_calls"), - message.get("reasoning_content"), - message.get("thinking_blocks"), - ) - def _restore_runtime_checkpoint(self, session: Session) -> bool: """Materialize an unfinished turn into session history before a new request.""" - from datetime import datetime - - checkpoint = cast( - object, - session.metadata.get(self._RUNTIME_CHECKPOINT_KEY), - ) - if not isinstance(checkpoint, dict): - return False - checkpoint_data = cast(dict[str, Any], checkpoint) - - assistant_message = cast(object, checkpoint_data.get("assistant_message")) - completed_tool_results = cast( - Iterable[object], - checkpoint_data.get("completed_tool_results") or [], - ) - pending_tool_calls = cast( - Iterable[object], - checkpoint_data.get("pending_tool_calls") or [], - ) - - restored_messages: list[dict[str, Any]] = [] - if isinstance(assistant_message, dict): - restored = dict(cast(dict[str, Any], assistant_message)) - restored.setdefault("timestamp", datetime.now().isoformat()) - restored_messages.append(restored) - for message in completed_tool_results: - if isinstance(message, dict): - restored = dict(cast(dict[str, Any], message)) - restored.setdefault("timestamp", datetime.now().isoformat()) - restored_messages.append(restored) - for tool_call in pending_tool_calls: - if not isinstance(tool_call, dict): - continue - tool_call_data = cast(dict[str, Any], tool_call) - tool_id = tool_call_data.get("id") - function_data = cast( - dict[str, Any], - tool_call_data.get("function") or {}, - ) - name = function_data.get("name") or "tool" - restored_messages.append( - { - "role": "tool", - "tool_call_id": tool_id, - "name": name, - "content": "Error: Task interrupted before this tool finished.", - "timestamp": datetime.now().isoformat(), - } - ) - - overlap = 0 - max_overlap = min(len(session.messages), len(restored_messages)) - for size in range(max_overlap, 0, -1): - existing = session.messages[-size:] - restored = restored_messages[:size] - if all( - self._checkpoint_message_key(left) == self._checkpoint_message_key(right) - for left, right in zip(existing, restored) - ): - overlap = size - break - appended_messages = restored_messages[overlap:] - session.messages.extend(appended_messages) - assistant_message_data = ( - cast(dict[str, Any], assistant_message) - if isinstance(assistant_message, dict) - else None - ) - provider_state_is_synchronized = ( - checkpoint_data.get(self._PROVIDER_STATE_CHECKPOINT_VERSION_KEY) - == self._PROVIDER_STATE_CHECKPOINT_VERSION - ) - phase = checkpoint_data.get("phase") - exact_final_response = ( - phase == "final_response" - and assistant_message_data is not None - and assistant_message_data.get("role") == "assistant" - and not bool(checkpoint_data.get("completed_tool_results")) - and not bool(checkpoint_data.get("pending_tool_calls")) - ) - exact_completed_tools = ( - phase == "tools_completed" - and assistant_message_data is not None - and assistant_message_data.get("role") == "assistant" - and not bool(checkpoint_data.get("pending_tool_calls")) - ) - if not ( - provider_state_is_synchronized - and (exact_final_response or exact_completed_tools) - ): - session.provider_state = None - - self._clear_pending_user_turn(session) - self._clear_runtime_checkpoint(session) - return True - - def _restore_pending_user_turn(self, session: Session) -> bool: - """Close a turn that only persisted the user message before crashing.""" - from datetime import datetime - - if not session.metadata.get(self._PENDING_USER_TURN_KEY): - return False - - if session.messages and session.messages[-1].get("role") == "user": - session.messages.append( - { - "role": "assistant", - "content": "Error: Task interrupted before a response was generated.", - "timestamp": datetime.now().isoformat(), - } - ) - session.provider_state = None - session.updated_at = datetime.now() - - self._clear_pending_user_turn(session) - return True + return restore_runtime_checkpoint(session) async def process_direct( self, diff --git a/nanobot/agent/runner.py b/nanobot/agent/runner.py index fddc4bbba..380f6a792 100644 --- a/nanobot/agent/runner.py +++ b/nanobot/agent/runner.py @@ -37,6 +37,7 @@ from nanobot.runtime_context import ( reattach_runtime_context, ) from nanobot.session.history_visibility import is_hidden_history_message +from nanobot.session.recovery import PENDING_FOLLOWUP_ID_KEY from nanobot.utils.helpers import ( IncrementalThinkExtractor, build_assistant_message, @@ -234,6 +235,23 @@ class AgentRunner: merged.get("content"), injection.get("content"), ) + followup_id = injection.get(PENDING_FOLLOWUP_ID_KEY) + if isinstance(followup_id, str) and followup_id: + existing = cast(object, merged.get(PENDING_FOLLOWUP_ID_KEY)) + followup_ids = ( + [existing] + if isinstance(existing, str) + else [ + item + for item in cast(list[object], existing) + if isinstance(item, str) + ] + if isinstance(existing, list) + else [] + ) + if followup_id not in followup_ids: + followup_ids.append(followup_id) + merged[PENDING_FOLLOWUP_ID_KEY] = followup_ids messages[-1] = merged continue messages.append(injection) diff --git a/nanobot/bus/outbound_events.py b/nanobot/bus/outbound_events.py index 7bbe1949b..82ab3ed9f 100644 --- a/nanobot/bus/outbound_events.py +++ b/nanobot/bus/outbound_events.py @@ -62,6 +62,15 @@ class TurnEndEvent(OutboundEvent): context_window_tokens: int | None = None +@dataclass(frozen=True) +class RecoveryStateEvent(OutboundEvent): + status: str + recovery_id: str + reason: str | None = None + attempts: int = 0 + can_continue: bool | None = None + + @dataclass(frozen=True) class GoalStatusEvent(OutboundEvent): status: str diff --git a/nanobot/channels/manager.py b/nanobot/channels/manager.py index 90c7506e6..4b16774a8 100644 --- a/nanobot/channels/manager.py +++ b/nanobot/channels/manager.py @@ -104,6 +104,9 @@ class ChannelManager: webui_mcp_runtime_status: Callable[[], Mapping[str, str]] | None = None, webui_mcp_reload: Callable[[], Awaitable[dict[str, Any]]] | None = None, webui_skill_state_action: Callable[[set[str]], None] | None = None, + webui_recovery_action: ( + Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] | None + ) = None, config_path: Path | None = None, ): if config_path is None: @@ -126,6 +129,7 @@ class ChannelManager: self._webui_mcp_runtime_status = webui_mcp_runtime_status self._webui_mcp_reload = webui_mcp_reload self._webui_skill_state_action = webui_skill_state_action + self._webui_recovery_action = webui_recovery_action self.channels: dict[str, BaseChannel] = {} self._channel_owners: dict[str, str] = {} self._channel_runtime_specs: dict[str, tuple[str, str]] = {} @@ -197,6 +201,7 @@ class ChannelManager: mcp_runtime_status=self._webui_mcp_runtime_status, mcp_reload=self._webui_mcp_reload, skill_state_action=self._webui_skill_state_action, + recovery_action=self._webui_recovery_action, logger=logger, ) kwargs["gateway"] = gateway @@ -615,6 +620,12 @@ class ChannelManager: if target is None: logger.warning("Restart notice target channel is not enabled: {}", notice.channel) return + if notice.channel == "websocket": + # Reconnect and recovery are already represented by WebSocket + # protocol state. A generic restart-complete notice must not + # masquerade as a recovery transition and overwrite a real + # awaiting-user checkpoint in connected clients. + return while not target.is_running: remaining = deadline - loop.time() diff --git a/nanobot/channels/websocket/runtime.py b/nanobot/channels/websocket/runtime.py index 88be1f76b..58fb0d818 100644 --- a/nanobot/channels/websocket/runtime.py +++ b/nanobot/channels/websocket/runtime.py @@ -32,6 +32,7 @@ from nanobot.bus.outbound_events import ( GoalStateSyncEvent, GoalStatusEvent, ProgressEvent, + RecoveryStateEvent, RuntimeModelUpdatedEvent, SessionUpdatedEvent, TurnEndEvent, @@ -55,6 +56,7 @@ from nanobot.security.workspace_access import ( ) from nanobot.session.goal_state import goal_state_ws_blob from nanobot.session.model_selection import model_preset_from_metadata +from nanobot.session.recovery import recovery_state_from_metadata from nanobot.session.webui_turns import ( clear_websocket_turn_if_current, clear_websocket_turns, @@ -453,6 +455,9 @@ class WebSocketChannel(BaseChannel): self.logger.warning("ignoring invalid model preset metadata for chat_id={}", chat_id) fields["model_preset"] = None if isinstance(metadata, dict): + recovery_state = recovery_state_from_metadata(metadata) + if recovery_state is not None: + fields["recovery_state"] = recovery_state usage = metadata.get("_last_usage") if isinstance(usage, dict): sanitized_usage: dict[str, int | float] = {} @@ -1740,6 +1745,10 @@ class WebSocketChannel(BaseChannel): provenance=event.provenance, ) return + if isinstance(event, RecoveryStateEvent): + if conns: + await self.send_recovery_state(msg.chat_id, event) + return if isinstance(event, GoalStateSyncEvent): if conns: await self.send_goal_state(msg.chat_id, event.goal_state or {"active": False}) @@ -2057,6 +2066,27 @@ class WebSocketChannel(BaseChannel): for connection in conns: await self._safe_send_to(connection, raw, label=" turn_end ") + async def send_recovery_state( + self, + chat_id: str, + event: RecoveryStateEvent, + ) -> None: + """Publish one structured recovery transition without chat pollution.""" + body: dict[str, Any] = { + "event": "recovery_state", + "chat_id": chat_id, + "status": event.status, + "recovery_id": event.recovery_id, + "attempts": event.attempts, + } + if event.reason: + body["reason"] = event.reason + if event.can_continue is not None: + body["can_continue"] = event.can_continue + raw = json.dumps(body, ensure_ascii=False) + for connection in list(self._subs.get(chat_id, ())): + await self._safe_send_to(connection, raw, label=" recovery_state ") + async def send_goal_state(self, chat_id: str, blob: dict[str, Any]) -> None: """Push persisted goal-state snapshot for *chat_id* (multi-chat isolation).""" conns = list(self._subs.get(chat_id, ())) diff --git a/nanobot/channels/websocket/tests/test_websocket_channel.py b/nanobot/channels/websocket/tests/test_websocket_channel.py index 8be1c16c0..2552d777f 100644 --- a/nanobot/channels/websocket/tests/test_websocket_channel.py +++ b/nanobot/channels/websocket/tests/test_websocket_channel.py @@ -27,6 +27,7 @@ from nanobot.bus.outbound_events import ( GoalStateSyncEvent, GoalStatusEvent, ProgressEvent, + RecoveryStateEvent, RuntimeModelUpdatedEvent, SessionUpdatedEvent, TurnEndEvent, @@ -2720,6 +2721,39 @@ async def test_send_turn_end_emits_turn_end_event() -> None: ] +@pytest.mark.asyncio +async def test_recovery_state_is_a_structured_event_not_assistant_text() -> None: + bus = MagicMock() + channel = WebSocketChannel( + {"enabled": True, "allowFrom": ["*"]}, + bus, + gateway=_basic_handler(bus), + ) + mock_ws = AsyncMock() + channel._attach(mock_ws, "chat-1") + + await channel.send(OutboundMessage( + channel="websocket", + chat_id="chat-1", + content="", + event=RecoveryStateEvent( + status="awaiting_user", + recovery_id="recovery-1", + reason="tool_state_unknown", + attempts=1, + ), + )) + + assert _sent_ws_payloads(mock_ws) == [{ + "event": "recovery_state", + "chat_id": "chat-1", + "status": "awaiting_user", + "recovery_id": "recovery-1", + "reason": "tool_state_unknown", + "attempts": 1, + }] + + @pytest.mark.asyncio async def test_system_command_turn_end_only_refreshes_session_metadata() -> None: bus = MagicMock() diff --git a/nanobot/channels/websocket/tests/test_websocket_http_routes.py b/nanobot/channels/websocket/tests/test_websocket_http_routes.py index 5e1566cc2..7db009867 100644 --- a/nanobot/channels/websocket/tests/test_websocket_http_routes.py +++ b/nanobot/channels/websocket/tests/test_websocket_http_routes.py @@ -83,6 +83,7 @@ def _make_handler( channel_feature_action: Any | None = None, channel_runtime_status: Any | None = None, mcp_reload: Any | None = None, + recovery_action: Any | None = None, ) -> GatewayServices: config = WebSocketConfig.model_validate(cfg) if isinstance(cfg, dict) else cfg workspace = workspace_path or Path.cwd() @@ -103,6 +104,7 @@ def _make_handler( channel_feature_action=channel_feature_action, channel_runtime_status=channel_runtime_status, mcp_reload=mcp_reload, + recovery_action=recovery_action, ) @@ -121,6 +123,7 @@ def _ch( channel_feature_action: Any | None = None, channel_runtime_status: Any | None = None, mcp_reload: Any | None = None, + recovery_action: Any | None = None, **extra: Any, ) -> WebSocketChannel: cfg: dict[str, Any] = { @@ -145,6 +148,7 @@ def _ch( channel_feature_action=channel_feature_action, channel_runtime_status=channel_runtime_status, mcp_reload=mcp_reload, + recovery_action=recovery_action, ) return InProcessHttpChannel(cfg, bus, gateway=gateway) @@ -3242,6 +3246,28 @@ async def _webui_mutate( ) +@pytest.mark.asyncio +async def test_recovery_mutation_uses_authenticated_websocket_action(bus: MagicMock) -> None: + recovery_action = AsyncMock(return_value={ + "status": "resuming", + "recovery_id": "recovery-1", + }) + channel = _ch(bus, recovery_action=recovery_action) + + response = await _webui_mutate( + channel, + "recovery.continue", + {"chat_id": "chat-1", "recovery_id": "recovery-1"}, + ) + + assert response.status_code == 200 + assert response.json()["status"] == "resuming" + recovery_action.assert_awaited_once_with( + "continue", + {"chat_id": "chat-1", "recovery_id": "recovery-1"}, + ) + + @pytest.mark.asyncio async def test_workspace_folder_picker_is_local_authenticated_mutation( bus: MagicMock, diff --git a/nanobot/cli/gateway_runtime.py b/nanobot/cli/gateway_runtime.py index 971b9a7ab..55239ed07 100644 --- a/nanobot/cli/gateway_runtime.py +++ b/nanobot/cli/gateway_runtime.py @@ -322,6 +322,7 @@ def _run_gateway( from nanobot.providers.fallback_provider import FallbackProvider from nanobot.providers.image_generation import image_gen_provider_configs from nanobot.session.manager import SessionManager + from nanobot.session.recovery import RecoveryCoordinator from nanobot.session.webui_turns import ( WebuiTurnCoordinator, WebuiTurnRoutePolicy, @@ -422,6 +423,12 @@ def _run_gateway( tools = ToolRegistry() mcp_provider = MCPProvider.from_config(config, tools) + recovery = RecoveryCoordinator( + sessions=session_manager, + bus=bus, + unified_session=config.agents.defaults.unified_session, + ) + # Create agent with cron service agent = AgentLoop.from_config( config, bus, @@ -440,6 +447,7 @@ def _run_gateway( local_trigger_store=trigger_store, hook_factories=[create_file_edit_activity_hook], tool_registry=tools, + recovery_admission=recovery, ) def _schedule_webui_background(awaitable: Awaitable[None]) -> None: agent.schedule_background(cast(Coroutine[Any, Any, None], awaitable)) @@ -448,6 +456,7 @@ def _run_gateway( bus=bus, sessions=session_manager, schedule_background=_schedule_webui_background, + recovery=recovery, ) webui_turn_coordinator.subscribe(runtime_events) from nanobot.bus.events import OutboundMessage @@ -683,6 +692,7 @@ def _run_gateway( webui_mcp_runtime_status=mcp_provider.runtime_status, webui_mcp_reload=mcp_provider.reload, webui_skill_state_action=_webui_skill_state_action, + webui_recovery_action=recovery.handle_action, config_path=Path(config_path), ) @@ -849,6 +859,7 @@ def _run_gateway( tasks: list[asyncio.Task[Any]] = [] shutdown_task: asyncio.Task[Any] | None = None runtime_tasks: asyncio.Future[list[Any]] | None = None + startup_complete = False shutdown_event = asyncio.Event() cli_terminal._ensure_interactive_tty_mode() restore_shutdown_handlers = _install_gateway_shutdown_handlers( @@ -861,6 +872,10 @@ def _run_gateway( await cron.start() # Re-read once on first admission to close the watcher subscription window. agent.runtime_resolver.invalidate() + # Recovery must finish before WebSocket and other channels begin + # accepting new input. That makes a new user message reliably + # supersede an old recoverable turn instead of racing its queue. + await recovery.scan() async def _run_agent() -> None: try: await mcp_provider.connect() @@ -915,6 +930,7 @@ def _run_gateway( name="nanobot-webui-dev-server", )) runtime_tasks = asyncio.gather(*tasks) + startup_complete = True shutdown_task = asyncio.create_task( shutdown_event.wait(), name="nanobot-gateway-shutdown", @@ -936,6 +952,10 @@ def _run_gateway( console.print("\n[red]Error: Gateway crashed unexpectedly[/red]") console.print(traceback.format_exc()) + if not startup_complete: + # Do not report a successful gateway command when startup + # failed before any runtime task or listener was created. + raise typer.Exit(1) finally: try: if shutdown_task and not shutdown_task.done(): @@ -943,6 +963,8 @@ def _run_gateway( with suppress(asyncio.CancelledError): await shutdown_task cron.stop() + if gateway_runtime.preserves_inflight_turns_on_exit(): + agent.preserve_inflight_turns_on_shutdown() agent.stop() # Cancel runtime tasks first, then deterministically close # exec/MCP resources while the event loop is still alive. diff --git a/nanobot/gateway/runtime.py b/nanobot/gateway/runtime.py index d0f6fb1fb..8c3ad86ff 100644 --- a/nanobot/gateway/runtime.py +++ b/nanobot/gateway/runtime.py @@ -201,6 +201,52 @@ class GatewayRuntime(ManagedProcessRuntime[ProcessStartOptions]): """Serialize long lifecycle transitions without blocking child cleanup.""" return FileLock(f"{self.paths.state_path}.transition.lock") + @property + def _restart_intent_path(self) -> Path: + """Return the short-lived marker used to distinguish restart from stop. + + A gateway receives the same operating-system termination request for a + graceful ``restart`` and an explicit ``stop``. The marker lets the + exiting process preserve its durable turn checkpoint only for the + former. It is intentionally local to one gateway instance. + """ + return self.paths.state_path.with_name(f"{self.paths.state_path.name}.restart") + + def preserves_inflight_turns_on_exit(self) -> bool: + """Whether this gateway was asked to exit as part of a managed restart.""" + try: + raw_intent: object = json.loads( + self._restart_intent_path.read_text(encoding="utf-8") + ) + except (json.JSONDecodeError, OSError): + return False + if not isinstance(raw_intent, dict): + return False + intent = cast(dict[str, object], raw_intent) + return intent.get("pid") == os.getpid() + + def _write_restart_intent(self, pid: int) -> None: + self.paths.run_dir.mkdir(parents=True, exist_ok=True) + target = self._restart_intent_path + with tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + dir=target.parent, + prefix=f".{target.name}.", + delete=False, + ) as handle: + json.dump({"pid": pid}, handle) + handle.flush() + os.fsync(handle.fileno()) + temporary = Path(handle.name) + os.replace(temporary, target) + + def _clear_restart_intent(self) -> None: + try: + self._restart_intent_path.unlink() + except FileNotFoundError: + pass + def start_background(self, options: ProcessStartOptions) -> RuntimeResult: """Start the gateway detached from the current terminal.""" lease = GatewayClientLease(self, kind="gateway-background") @@ -353,7 +399,15 @@ class GatewayRuntime(ManagedProcessRuntime[ProcessStartOptions]): "gateway_foreground_restart_required", status, ) - stop_result = self._stop(timeout_s=timeout_s) + assert status.pid is not None + self._write_restart_intent(status.pid) + try: + stop_result = self._stop(timeout_s=timeout_s) + finally: + # The old process reads the marker while handling shutdown. + # Never let a stale marker turn a later explicit stop into a + # recoverable restart. + self._clear_restart_intent() if not stop_result.ok: return self._result(stop_result) with self._lifecycle_lock(): diff --git a/nanobot/session/manager.py b/nanobot/session/manager.py index e7d07c433..8c919c3a5 100644 --- a/nanobot/session/manager.py +++ b/nanobot/session/manager.py @@ -48,15 +48,21 @@ _SESSION_PREVIEW_MAX_CHARS = 120 _SESSION_LIST_PREVIEW_MAX_RECORDS = 200 _SESSION_LIST_PREVIEW_MAX_CHARS = 1_000_000 _SESSION_DATA_ERRORS = (ValueError, TypeError, AttributeError, KeyError) +_RUNTIME_CHECKPOINT_DATA_ERRORS = (OSError, *_SESSION_DATA_ERRORS) _PROVIDER_STATE_RECORD_TYPE = "provider_state" _PROVIDER_STATE_RECORD_PREFIX_RE = re.compile( r'^\s*\{\s*"_type"\s*:\s*"provider_state"\s*(?:,|\})' ) +_RUNTIME_CHECKPOINT_KEY = "runtime_checkpoint" +_RUNTIME_CHECKPOINT_VERSION = 1 +_RUNTIME_CHECKPOINT_SUFFIX = ".checkpoint.json" _FORK_VOLATILE_METADATA_KEYS = { "goal_state", "pending_user_turn", + "pending_user_followups", "runtime_checkpoint", "session_handle", + "webui_recovery", "thread_goal", "title", "title_user_edited", @@ -1001,6 +1007,9 @@ class JsonlSessionStore: def get_session_path(self, key: str) -> Path: return self.sessions_dir / f"{self.storage_key(key)}.jsonl" + def get_runtime_checkpoint_path(self, key: str) -> Path: + return self.sessions_dir / f"{self.storage_key(key)}{_RUNTIME_CHECKPOINT_SUFFIX}" + def get_legacy_lossy_path(self, key: str) -> Path: return self.sessions_dir / f"{safe_filename(key.replace(':', '_'))}.jsonl" @@ -1066,7 +1075,7 @@ class JsonlSessionStore: else: messages.append(data) - return Session( + session = Session( key=key, messages=messages, created_at=created_at or datetime.now(), @@ -1075,6 +1084,8 @@ class JsonlSessionStore: last_consolidated=last_consolidated, provider_state=provider_state, ) + self._overlay_runtime_checkpoint_unlocked(session, path) + return session except _SESSION_DATA_ERRORS as e: logger.warning("Failed to load session {}: {}", key, e) repaired = self._repair_unlocked(key) @@ -1159,7 +1170,7 @@ class JsonlSessionStore: if not messages and not metadata and provider_state is None: return None - return Session( + session = Session( key=key, messages=messages, created_at=created_at or datetime.now(), @@ -1168,6 +1179,8 @@ class JsonlSessionStore: last_consolidated=last_consolidated, provider_state=provider_state, ) + self._overlay_runtime_checkpoint_unlocked(session, path) + return session except _SESSION_DATA_ERRORS as e: logger.warning("Repair failed for session {}: {}", key, e) return None @@ -1186,6 +1199,105 @@ class JsonlSessionStore: with self._session_files_lock: self._save_unlocked(session, fsync=fsync) + def save_runtime_checkpoint(self, session: Session) -> None: + """Atomically persist only the volatile in-flight turn state. + + A checkpoint is written several times during a tool-heavy turn. Keeping it + beside the append history avoids copying the full transcript at each safe + recovery boundary. + """ + with self._session_files_lock: + path = self.get_session_path(session.key) + if not path.exists(): + # A user turn normally creates the session first. Internal callers + # may checkpoint a fresh session, so establish the durable base once. + self._save_unlocked(session) + return + + checkpoint = session.metadata.get(_RUNTIME_CHECKPOINT_KEY) + if not isinstance(checkpoint, dict): + self.get_runtime_checkpoint_path(session.key).unlink(missing_ok=True) + return + + payload: dict[str, Any] = { + "version": _RUNTIME_CHECKPOINT_VERSION, + "session_key": session.key, + "base_updated_at": session.updated_at.isoformat(), + "base_message_count": len(session.messages), + "checkpoint": checkpoint, + "provider_state": ( + session.provider_state.to_private_record() + if session.provider_state is not None + else None + ), + } + target = self.get_runtime_checkpoint_path(session.key) + tmp = target.with_name(f".{target.name}.{secrets.token_hex(8)}.tmp") + try: + with open(tmp, "x", encoding="utf-8") as handle: + os.chmod(tmp, 0o600) + json.dump( + payload, + handle, + ensure_ascii=False, + separators=(",", ":"), + ) + os.replace(tmp, target) + finally: + tmp.unlink(missing_ok=True) + + def _overlay_runtime_checkpoint_unlocked(self, session: Session, main_path: Path) -> None: + checkpoint_path = self.get_runtime_checkpoint_path(session.key) + try: + checkpoint_stat = checkpoint_path.lstat() + if not stat.S_ISREG(checkpoint_stat.st_mode): + logger.warning( + "Ignoring non-regular runtime checkpoint for session {}", + session.key, + ) + return + # A complete session save supersedes an older sidecar. This comparison + # closes the small crash window between replacing the JSONL and unlinking + # its previous checkpoint. + if main_path.stat().st_mtime_ns > checkpoint_stat.st_mtime_ns: + checkpoint_path.unlink(missing_ok=True) + return + raw = _json_object(json.loads(checkpoint_path.read_text(encoding="utf-8"))) + if ( + raw.get("version") != _RUNTIME_CHECKPOINT_VERSION + or raw.get("session_key") != session.key + or raw.get("base_updated_at") != session.updated_at.isoformat() + or raw.get("base_message_count") != len(session.messages) + or not isinstance(raw.get("checkpoint"), dict) + ): + checkpoint_path.unlink(missing_ok=True) + return + provider_record = raw.get("provider_state") + provider_state = ( + None + if provider_record is None + else ProviderConversationState.from_private_record(provider_record) + ) + if provider_record is not None and provider_state is None: + raise ValueError("invalid checkpoint provider state") + session.metadata[_RUNTIME_CHECKPOINT_KEY] = cast( + dict[str, Any], raw["checkpoint"] + ) + session.provider_state = provider_state + except FileNotFoundError: + return + except _RUNTIME_CHECKPOINT_DATA_ERRORS as exc: + logger.warning( + "Ignoring invalid runtime checkpoint for session {}: {}", + session.key, + exc, + ) + # Atomic writes mean a malformed target cannot become valid later. + # Remove it once so future loads do not repeatedly parse and log it. + with suppress(OSError): + if checkpoint_path.is_file() and not checkpoint_path.is_symlink(): + checkpoint_path.unlink() + def _save_unlocked(self, session: Session, *, fsync: bool = False) -> None: path = self.get_session_path(session.key) tmp_path = path.with_name(f".{path.name}.{secrets.token_hex(8)}.tmp") @@ -1215,6 +1327,10 @@ class JsonlSessionStore: os.replace(tmp_path, path) + # The full record now contains the authoritative checkpoint state (or + # its removal), so an older volatile overlay is no longer needed. + self.get_runtime_checkpoint_path(session.key).unlink(missing_ok=True) + if fsync: with suppress(PermissionError): fd = os.open(str(path.parent), os.O_RDONLY) @@ -1278,6 +1394,7 @@ class JsonlSessionStore: def _delete_unlocked(self, key: str) -> bool: paths = [ self.get_session_path(key), + self.get_runtime_checkpoint_path(key), self.get_legacy_lossy_path(key), self.get_legacy_session_path(key), ] @@ -1585,6 +1702,10 @@ class SessionManager: """Get the collision-resistant workspace path for a session.""" return self._jsonl_store.get_session_path(key) + def _get_runtime_checkpoint_path(self, key: str) -> Path: + """Get the private in-flight checkpoint path for a session.""" + return self._jsonl_store.get_runtime_checkpoint_path(key) + def _get_legacy_lossy_path(self, key: str) -> Path: """Previous workspace session path using lossy ':' to '_' replacement.""" return self._jsonl_store.get_legacy_lossy_path(key) @@ -1653,6 +1774,18 @@ class SessionManager: self._store.save(session, fsync=fsync) self._remember(session) + def save_runtime_checkpoint(self, session: Session) -> None: + """Persist volatile recovery state without rewriting long history.""" + if not session.policy.persist: + return + if self._store is self._jsonl_store: + self._jsonl_store.save_runtime_checkpoint(session) + self._remember(session) + return + # Third-party stores keep their existing all-or-nothing semantics until + # they opt into a dedicated checkpoint primitive. + self.save(session) + def rename_model_preset(self, old_name: str, new_name: str) -> int: """Rename a session-scoped model preset across durable and live sessions.""" if old_name == new_name: diff --git a/nanobot/session/recovery.py b/nanobot/session/recovery.py new file mode 100644 index 000000000..7a93a83ea --- /dev/null +++ b/nanobot/session/recovery.py @@ -0,0 +1,911 @@ +"""Durable, side-effect-safe recovery for interrupted WebUI turns. + +The coordinator owns restart policy. AgentLoop only exposes checkpoint +materialization and an admission hook, so transport code never has to guess +whether an interrupted tool call is safe to replay. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import json +from collections.abc import Iterable, Mapping +from datetime import datetime +from typing import Any, Protocol, cast +from uuid import uuid4 + +from loguru import logger + +from nanobot.bus.events import InboundMessage +from nanobot.bus.outbound_events import ( + RecoveryStateEvent, + SessionUpdatedEvent, + outbound_message_for_event, +) +from nanobot.bus.queue import MessageBus +from nanobot.session import turn_continuation +from nanobot.session.keys import UNIFIED_SESSION_KEY, last_channel_from_metadata +from nanobot.session.manager import Session, SessionManager +from nanobot.webui.metadata import WEBUI_TURN_METADATA_KEY + +RUNTIME_CHECKPOINT_KEY = "runtime_checkpoint" +PENDING_USER_TURN_KEY = "pending_user_turn" +RECOVERY_METADATA_KEY = "webui_recovery" +RECOVERY_INBOUND_METADATA_KEY = "_webui_recovery_id" +PENDING_FOLLOWUPS_KEY = "pending_user_followups" +PENDING_FOLLOWUP_ID_KEY = "_recovery_followup_id" +PROVIDER_STATE_CHECKPOINT_VERSION_KEY = "provider_state_checkpoint_version" +PROVIDER_STATE_CHECKPOINT_VERSION = "v1" + +_RECOVERY_STATUSES = frozenset({"resuming", "awaiting_user", "recovered", "failed"}) +_UNCERTAIN_TOOL_PHASES = frozenset({"awaiting_tools"}) +_KNOWN_CHECKPOINT_PHASES = frozenset( + {"final_response", "tools_completed", "awaiting_tools", "error"} +) + + +class RecoveryActionError(ValueError): + """A stale or malformed recovery action from an authenticated WebUI.""" + + def __init__(self, message: str, *, status: int = 400) -> None: + super().__init__(message) + self.status = status + + +class RecoveryAdmission(Protocol): + """Narrow AgentLoop boundary for explicit recovery validation.""" + + async def admit(self, message: InboundMessage) -> bool: ... + + def register_recovery_task(self, session_key: str, task: asyncio.Task[Any]) -> None: ... + + def unregister_recovery_task(self, session_key: str, task: asyncio.Task[Any]) -> None: ... + + +def record_pending_followup(session: Session, message: InboundMessage) -> str | None: + """Durably journal a WebUI follow-up before injecting it into a live turn.""" + if message.channel != "websocket": + return None + try: + metadata_value: object = json.loads(json.dumps(message.metadata)) + except (TypeError, ValueError): + logger.warning("Skipping non-serializable WebUI follow-up for recovery") + return None + if not isinstance(metadata_value, dict): + return None + metadata = cast(dict[str, Any], metadata_value) + existing_id = metadata.pop(PENDING_FOLLOWUP_ID_KEY, None) + followup_id = ( + existing_id + if isinstance(existing_id, str) and existing_id + else uuid4().hex + ) + records = _pending_followup_records(session) + if any(record.get("id") == followup_id for record in records): + return followup_id + records.append( + { + "id": followup_id, + "sender_id": message.sender_id, + "chat_id": message.chat_id, + "content": message.content, + "media": list(message.media or []), + "metadata": metadata, + } + ) + # This journal is the recovery source of truth, not a mirror of the + # bounded in-memory injection queue. A queued turn can receive more + # follow-ups than the live queue accepts; dropping older journal entries + # would make those acknowledged user messages unrecoverable after a + # gateway restart. Entries are removed only once their user rows are + # committed by ``acknowledge_pending_followups``. + session.metadata[PENDING_FOLLOWUPS_KEY] = records + session.updated_at = datetime.now() + return followup_id + + +def pending_followups(session: Session) -> list[InboundMessage]: + """Decode still-unacknowledged follow-ups from durable session metadata.""" + messages: list[InboundMessage] = [] + for record in _pending_followup_records(session): + followup_id = cast(object, record.get("id")) + sender_id = cast(object, record.get("sender_id")) + chat_id = cast(object, record.get("chat_id")) + content = cast(object, record.get("content")) + metadata = cast(object, record.get("metadata")) + if ( + not isinstance(followup_id, str) + or not followup_id + or not isinstance(sender_id, str) + or not sender_id + or not isinstance(chat_id, str) + or not chat_id + ): + continue + if not isinstance(content, str) or not isinstance(metadata, dict): + continue + media_value = cast(object, record.get("media")) + media = ( + [item for item in cast(list[object], media_value) if isinstance(item, str)] + if isinstance(media_value, list) + else [] + ) + messages.append( + InboundMessage( + channel="websocket", + sender_id=sender_id, + chat_id=chat_id, + content=content, + media=media, + metadata={**cast(dict[str, Any], metadata), PENDING_FOLLOWUP_ID_KEY: followup_id}, + session_key_override=session.key, + require_existing_session=True, + ) + ) + return messages + + +def acknowledge_pending_followups(session: Session, followup_ids: Iterable[str]) -> None: + """Remove journal entries whose user rows were committed to history.""" + acknowledged = set(followup_ids) + if not acknowledged: + return + records = [record for record in _pending_followup_records(session) if record.get("id") not in acknowledged] + if records: + session.metadata[PENDING_FOLLOWUPS_KEY] = records + else: + session.metadata.pop(PENDING_FOLLOWUPS_KEY, None) + + +def _pending_followup_records(session: Session) -> list[dict[str, Any]]: + raw = cast(object, session.metadata.get(PENDING_FOLLOWUPS_KEY)) + if not isinstance(raw, list): + return [] + values = cast(list[object], raw) + return [cast(dict[str, Any], value) for value in values if isinstance(value, dict)] + + +def _checkpoint_message_key(message: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + message.get("role"), + message.get("content"), + message.get("tool_call_id"), + message.get("name"), + message.get("tool_calls"), + message.get("reasoning_content"), + message.get("thinking_blocks"), + ) + + +def _checkpoint_tool_call_ids( + value: object, + *, + result_rows: bool = False, +) -> list[str] | None: + """Validate checkpoint tool rows and return their stable IDs.""" + if not isinstance(value, list): + return None + ids: list[str] = [] + for raw in cast(list[object], value): + if not isinstance(raw, dict): + return None + row = cast(dict[str, Any], raw) + id_key = "tool_call_id" if result_rows else "id" + call_id = cast(object, row.get(id_key)) + if not isinstance(call_id, str) or not call_id: + return None + if result_rows: + if row.get("role") != "tool": + return None + else: + function_value = cast(object, row.get("function")) + if not isinstance(function_value, dict): + return None + function = cast(dict[str, Any], function_value) + name = cast(object, function.get("name")) + if not isinstance(name, str) or not name: + return None + ids.append(call_id) + return ids if len(ids) == len(set(ids)) else None + + +def _runtime_checkpoint_is_well_formed(checkpoint: Mapping[str, Any]) -> bool: + """Return whether a checkpoint is safe to offer for continuation. + + Restoration stays tolerant so Dismiss can always clear corrupt state. + Continue is stricter: silently dropping a malformed tool result could make + the model repeat an external side effect. + """ + assistant_value = cast(object, checkpoint.get("assistant_message")) + if not isinstance(assistant_value, dict): + return False + assistant = cast(dict[str, Any], assistant_value) + if assistant.get("role") != "assistant": + return False + + completed_ids = _checkpoint_tool_call_ids( + cast(object, checkpoint.get("completed_tool_results")), + result_rows=True, + ) + pending_ids = _checkpoint_tool_call_ids( + cast(object, checkpoint.get("pending_tool_calls")), + ) + if completed_ids is None or pending_ids is None: + return False + assistant_calls_value = cast(object, assistant.get("tool_calls")) + assistant_call_ids = ( + [] + if assistant_calls_value is None + else _checkpoint_tool_call_ids(assistant_calls_value) + ) + if assistant_call_ids is None: + return False + + phase = checkpoint.get("phase") + if phase == "final_response": + content = cast(object, assistant.get("content")) + return ( + isinstance(content, str) + and bool(content.strip()) + and not assistant_call_ids + and not completed_ids + and not pending_ids + ) + if phase == "awaiting_tools": + return ( + bool(assistant_call_ids) + and not completed_ids + and len(assistant_call_ids) == len(pending_ids) + and set(assistant_call_ids) == set(pending_ids) + ) + if phase == "tools_completed": + return ( + bool(assistant_call_ids) + and not pending_ids + and len(assistant_call_ids) == len(completed_ids) + and set(assistant_call_ids) == set(completed_ids) + ) + # Error checkpoints have no current producer contract. Treat legacy or + # future instances as review-only until their exact persisted shape is + # specified; guessing here could make a partial side effect repeat. + return False + + +def restore_runtime_checkpoint(session: Session) -> bool: + """Materialize the durable checkpoint exactly once and clear it. + + Pending tool calls become explicit interrupted tool results. They are + never executed here. Provider-native state is retained only for the two + checkpoint shapes known to be synchronized with persisted history. + """ + checkpoint = cast(object, session.metadata.get(RUNTIME_CHECKPOINT_KEY)) + if not isinstance(checkpoint, dict): + return False + data = cast(dict[str, Any], checkpoint) + assistant = cast(object, data.get("assistant_message")) + completed_value = cast(object, data.get("completed_tool_results")) + pending_value = cast(object, data.get("pending_tool_calls")) + completed = cast(list[object], completed_value) if isinstance(completed_value, list) else [] + pending = cast(list[object], pending_value) if isinstance(pending_value, list) else [] + + restored: list[dict[str, Any]] = [] + if isinstance(assistant, dict): + assistant_row = cast(dict[str, Any], assistant) + else: + assistant_row = {} + if assistant_row.get("role") == "assistant": + row = dict(assistant_row) + row.setdefault("timestamp", datetime.now().isoformat()) + restored.append(row) + for value in completed: + if not isinstance(value, dict): + continue + tool_result = cast(dict[str, Any], value) + if tool_result.get("role") != "tool": + continue + row = dict(tool_result) + row.setdefault("timestamp", datetime.now().isoformat()) + restored.append(row) + for value in pending: + if not isinstance(value, dict): + continue + tool_call = cast(dict[str, Any], value) + tool_call_id = tool_call.get("id") + function_value = cast(object, tool_call.get("function")) + if not isinstance(tool_call_id, str) or not tool_call_id: + continue + function = ( + cast(dict[str, Any], function_value) + if isinstance(function_value, dict) + else {} + ) + name = function.get("name") + restored.append( + { + "role": "tool", + "tool_call_id": tool_call_id, + "name": name if isinstance(name, str) and name else "tool", + "content": "Error: Task interrupted before this tool finished.", + "timestamp": datetime.now().isoformat(), + "_recovery_interrupted": True, + } + ) + + overlap = 0 + for size in range(min(len(session.messages), len(restored)), 0, -1): + if all( + _checkpoint_message_key(left) == _checkpoint_message_key(right) + for left, right in zip(session.messages[-size:], restored[:size]) + ): + overlap = size + break + session.messages.extend(restored[overlap:]) + + assistant_data = cast(dict[str, Any], assistant) if isinstance(assistant, dict) else None + synchronized = ( + data.get(PROVIDER_STATE_CHECKPOINT_VERSION_KEY) + == PROVIDER_STATE_CHECKPOINT_VERSION + ) + phase = data.get("phase") + exact_final = ( + phase == "final_response" + and assistant_data is not None + and assistant_data.get("role") == "assistant" + and not data.get("completed_tool_results") + and not data.get("pending_tool_calls") + ) + exact_tools = ( + phase == "tools_completed" + and assistant_data is not None + and assistant_data.get("role") == "assistant" + and not data.get("pending_tool_calls") + ) + if not (synchronized and (exact_final or exact_tools)): + session.provider_state = None + + session.metadata.pop(PENDING_USER_TURN_KEY, None) + session.metadata.pop(RUNTIME_CHECKPOINT_KEY, None) + session.updated_at = datetime.now() + return True + + +def _discard_runtime_checkpoint(session: Session) -> bool: + """Drop checkpoint state that cannot be projected into valid history.""" + if RUNTIME_CHECKPOINT_KEY not in session.metadata: + return False + session.metadata.pop(RUNTIME_CHECKPOINT_KEY, None) + session.provider_state = None + session.updated_at = datetime.now() + return True + + +def restore_pending_interruption(session: Session, *, superseded: bool = False) -> bool: + """Close a persisted user-only turn without pretending it was answered.""" + if not session.metadata.get(PENDING_USER_TURN_KEY): + return False + if session.messages and session.messages[-1].get("role") == "user": + content = ( + "Task recovery was superseded by a newer message." + if superseded + else "Error: Task interrupted before a response was generated." + ) + session.messages.append( + { + "role": "assistant", + "content": content, + "timestamp": datetime.now().isoformat(), + "_recovery_interrupted": True, + } + ) + session.provider_state = None + session.updated_at = datetime.now() + session.metadata.pop(PENDING_USER_TURN_KEY, None) + return True + + +def append_recovery_interruption(session: Session, *, superseded: bool = False) -> None: + """Close a restored partial turn whose last durable row is not the user message.""" + if session.messages and session.messages[-1].get("_recovery_interrupted") is True: + return + session.messages.append( + { + "role": "assistant", + "content": ( + "Task recovery was superseded by a newer message." + if superseded + else "Error: Task recovery was interrupted before completion." + ), + "timestamp": datetime.now().isoformat(), + "_recovery_interrupted": True, + } + ) + session.provider_state = None + session.updated_at = datetime.now() + + +def recovery_state_from_metadata(metadata: Mapping[str, Any] | None) -> dict[str, Any] | None: + """Return a sanitized recovery state suitable for the WebSocket wire.""" + value = metadata.get(RECOVERY_METADATA_KEY) if metadata else None + if not isinstance(value, dict): + return None + state = cast(dict[str, Any], value) + status = state.get("status") + recovery_id = state.get("recovery_id") + if status not in _RECOVERY_STATUSES or not isinstance(recovery_id, str): + return None + payload: dict[str, Any] = {"status": status, "recovery_id": recovery_id} + reason = state.get("reason") + if isinstance(reason, str) and reason: + payload["reason"] = reason + attempts = state.get("attempts") + if isinstance(attempts, int) and attempts >= 0: + payload["attempts"] = attempts + can_continue = state.get("can_continue") + if isinstance(can_continue, bool): + payload["can_continue"] = can_continue + return payload + + +@dataclasses.dataclass(slots=True) +class RecoveryCoordinator: + """Classify, announce, and gate durable WebUI turn recovery.""" + + sessions: SessionManager + bus: MessageBus + unified_session: bool = False + _active_recovery_tasks: dict[str, asyncio.Task[Any]] = dataclasses.field( + default_factory=dict, + init=False, + repr=False, + ) + + def register_recovery_task(self, session_key: str, task: asyncio.Task[Any]) -> None: + """Track the task that owns an explicit recovery continuation.""" + self._active_recovery_tasks[session_key] = task + + def unregister_recovery_task(self, session_key: str, task: asyncio.Task[Any]) -> None: + """Drop a recovery task without removing a newer task for the same session.""" + if self._active_recovery_tasks.get(session_key) is task: + self._active_recovery_tasks.pop(session_key, None) + + async def _cancel_active_recovery(self, session_key: str) -> None: + """Stop an explicit continuation before accepting newer user input.""" + task = self._active_recovery_tasks.get(session_key) + if task is None or task is asyncio.current_task() or task.done(): + return + task.cancel() + # AgentLoop's cancellation path materializes any partial checkpoint and + # releases its pending queue. Wait for that ownership to be released + # before the newer message is routed. + await asyncio.gather(task, return_exceptions=True) + + async def scan(self) -> None: + """Recover every interrupted WebUI session once at gateway startup.""" + for key in self._recovery_candidates(): + metadata_payload = self.sessions.read_session_metadata(key) + raw_metadata = metadata_payload.get("metadata") if metadata_payload else None + metadata = cast(dict[str, Any], raw_metadata) if isinstance(raw_metadata, dict) else {} + route = self._websocket_route_for(key, metadata) + if route is None: + continue + unfinished = self._has_unfinished_webui_transcript(key) + if not self._needs_recovery(metadata) and not unfinished: + continue + session = self.sessions.get_or_create(key) + try: + await self._recover_session(session, route[1]) + await self._requeue_pending_followups(session) + except Exception: + logger.exception("failed to recover interrupted WebUI session {}", session.key) + state = recovery_state_from_metadata(session.metadata) + failed = self._set_state( + session, + status="failed", + recovery_id=cast(str, state["recovery_id"]) if state else uuid4().hex, + attempts=cast(int, state.get("attempts", 0)) if state else 0, + reason="recovery_failed", + can_continue=False, + ) + self.sessions.save(session) + await self._publish(route[1], failed) + + def _recovery_candidates(self) -> list[str]: + """Discover canonical and transcript-only WebUI sessions cheaply.""" + candidates = dict.fromkeys( + key + for item in self.sessions.list_sessions() + if isinstance((key := item.get("key")), str) + ) + try: + # Imported lazily because the sidebar index also projects recovery + # metadata. The index is the owner of transcript-only discovery; + # duplicating its filename and migration rules here would drift. + from nanobot.webui.session_list_index import list_webui_sessions + + for item in list_webui_sessions(self.sessions): + key = item.get("key") + if isinstance(key, str): + candidates.setdefault(key, None) + except Exception: + # Canonical checkpoint recovery remains available even if the + # optional display-history index is corrupt or unavailable. + logger.exception("failed to discover transcript-only WebUI sessions") + return list(candidates) + + @staticmethod + def _needs_recovery(metadata: Mapping[str, Any]) -> bool: + if metadata.get(PENDING_USER_TURN_KEY) is True: + return True + if isinstance(metadata.get(RUNTIME_CHECKPOINT_KEY), dict): + return True + followups = metadata.get(PENDING_FOLLOWUPS_KEY) + if isinstance(followups, list) and len(cast(list[object], followups)) > 0: + return True + state = recovery_state_from_metadata(metadata) + return bool(state and state["status"] in {"resuming", "awaiting_user", "failed"}) + + async def admit(self, message: InboundMessage) -> bool: + """Reject stale queued recoveries and let new user input supersede them.""" + recovery_id = message.metadata.get(RECOVERY_INBOUND_METADATA_KEY) + if isinstance(recovery_id, str): + session = self.sessions.get_or_create(message.session_key) + state = recovery_state_from_metadata(session.metadata) + return bool( + state + and state["status"] == "resuming" + and state["recovery_id"] == recovery_id + ) + if message.channel != "websocket": + return True + session = self.sessions.get_or_create(message.session_key) + state = recovery_state_from_metadata(session.metadata) + if state and state["status"] in {"resuming", "awaiting_user", "failed"}: + await self._cancel_active_recovery(message.session_key) + restore_runtime_checkpoint(session) + if not restore_pending_interruption(session, superseded=True): + append_recovery_interruption(session, superseded=True) + recovered = self._set_state( + session, + status="recovered", + recovery_id=cast(str, state["recovery_id"]), + attempts=cast(int, state.get("attempts", 0)), + reason="superseded", + ) + self.sessions.save(session) + await self._publish(message.chat_id, recovered) + return True + + async def turn_completed(self, session_key: str) -> None: + """Resolve a resuming state after the recovered turn commits.""" + session = self.sessions.get_or_create(session_key) + state = recovery_state_from_metadata(session.metadata) + if not state or state["status"] != "resuming": + return + route = self._websocket_route(session) + if route is None: + return + recovered = self._set_state( + session, + status="recovered", + recovery_id=cast(str, state["recovery_id"]), + attempts=cast(int, state.get("attempts", 0)), + reason="continued", + ) + self.sessions.save(session) + await self._publish(route[1], recovered) + + async def handle_action(self, action: str, payload: dict[str, Any]) -> dict[str, Any]: + """Apply an authenticated continue/dismiss operation.""" + chat_id = payload.get("chat_id") + recovery_id = payload.get("recovery_id") + if not isinstance(chat_id, str) or not chat_id: + raise RecoveryActionError("missing chat_id") + if not isinstance(recovery_id, str) or not recovery_id: + raise RecoveryActionError("missing recovery_id") + session = self.sessions.get_or_create(self._session_key(chat_id)) + state = recovery_state_from_metadata(session.metadata) + if not state or state["recovery_id"] != recovery_id: + raise RecoveryActionError("recovery state is stale", status=409) + + if action == "dismiss": + restore_runtime_checkpoint(session) + restore_pending_interruption(session) + next_state = self._set_state( + session, + status="recovered", + recovery_id=recovery_id, + attempts=cast(int, state.get("attempts", 0)), + reason="dismissed", + ) + self.sessions.save(session) + await self._publish(chat_id, next_state) + return next_state + if action != "continue": + raise RecoveryActionError("unknown recovery action") + if state["status"] not in {"awaiting_user", "failed"}: + raise RecoveryActionError("recovery is not waiting for confirmation", status=409) + if state.get("can_continue") is False: + raise RecoveryActionError("recovery context is unavailable", status=409) + next_state = self._set_state( + session, + status="resuming", + recovery_id=recovery_id, + attempts=cast(int, state.get("attempts", 0)) + 1, + reason="user_confirmed", + resume_message_count=len(session.messages), + ) + self.sessions.save(session) + await self._publish(chat_id, next_state) + await self._queue_continuation(session, chat_id, next_state) + return next_state + + async def _recover_session(self, session: Session, chat_id: str) -> None: + checkpoint_value = cast(object, session.metadata.get(RUNTIME_CHECKPOINT_KEY)) + checkpoint = ( + cast(dict[str, Any], checkpoint_value) + if isinstance(checkpoint_value, dict) + else None + ) + pending = session.metadata.get(PENDING_USER_TURN_KEY) is True + state = recovery_state_from_metadata(session.metadata) + if not pending and checkpoint is None: + if state and state["status"] == "resuming": + resume_count = self._resume_message_count(session) + if resume_count is not None and len(session.messages) > resume_count: + next_state = self._set_state( + session, + status="recovered", + recovery_id=cast(str, state["recovery_id"]), + attempts=cast(int, state.get("attempts", 0)), + reason="committed", + ) + else: + next_state = self._set_state( + session, + status="awaiting_user", + recovery_id=cast(str, state["recovery_id"]), + attempts=cast(int, state.get("attempts", 1)), + reason="loop_guard", + ) + self.sessions.save(session) + await self._publish(chat_id, next_state) + elif self._has_unfinished_webui_transcript(session.key): + # A normal last-client shutdown can materialize the checkpoint + # before the process exits. In that path there is no pending + # marker left to classify, but the append-only transcript still + # contains an activity row without a turn_end. Treat it as an + # interrupted turn instead of letting the UI resurrect it as a + # forever-running spinner. + waiting = self._set_state( + session, + status="awaiting_user", + recovery_id=uuid4().hex, + attempts=0, + reason="interrupted_without_checkpoint", + can_continue=False, + ) + self.sessions.save(session) + await self._publish(chat_id, waiting) + return + if state and state["status"] in {"awaiting_user", "failed"}: + await self._publish(chat_id, state) + return + if state and state["status"] == "resuming": + restore_runtime_checkpoint(session) + restore_pending_interruption(session) + waiting = self._set_state( + session, + status="awaiting_user", + recovery_id=cast(str, state["recovery_id"]), + attempts=cast(int, state.get("attempts", 1)), + reason="loop_guard", + ) + self.sessions.save(session) + await self._publish(chat_id, waiting) + return + + recovery_id = uuid4().hex + phase = checkpoint.get("phase") if checkpoint is not None else None + pending_calls = checkpoint.get("pending_tool_calls") if checkpoint is not None else None + if checkpoint is not None and phase not in _KNOWN_CHECKPOINT_PHASES: + _discard_runtime_checkpoint(session) + restore_pending_interruption(session) + waiting = self._set_state( + session, + status="awaiting_user", + recovery_id=recovery_id, + attempts=0, + reason="checkpoint_unknown", + can_continue=False, + ) + self.sessions.save(session) + await self._publish(chat_id, waiting) + return + if checkpoint is not None and not _runtime_checkpoint_is_well_formed(checkpoint): + _discard_runtime_checkpoint(session) + restore_pending_interruption(session) + waiting = self._set_state( + session, + status="awaiting_user", + recovery_id=recovery_id, + attempts=0, + reason="checkpoint_invalid", + can_continue=False, + ) + self.sessions.save(session) + await self._publish(chat_id, waiting) + return + if phase == "final_response": + restore_runtime_checkpoint(session) + recovered = self._set_state( + session, + status="recovered", + recovery_id=recovery_id, + attempts=0, + reason="answer_restored", + ) + self.sessions.save(session) + await self._publish(chat_id, recovered) + return + if phase in _UNCERTAIN_TOOL_PHASES or pending_calls: + restore_runtime_checkpoint(session) + waiting = self._set_state( + session, + status="awaiting_user", + recovery_id=recovery_id, + attempts=0, + reason="tool_state_unknown", + ) + self.sessions.save(session) + await self._publish(chat_id, waiting) + return + # A gateway restart is a lifecycle boundary. Never enqueue model work + # implicitly: even a synchronized checkpoint may sit next to an + # external side effect that the user should review first. The final + # answer path above only restores persisted output; it never executes. + restore_runtime_checkpoint(session) + waiting = self._set_state( + session, + status="awaiting_user", + recovery_id=recovery_id, + attempts=0, + reason="restart_requires_confirmation", + ) + self.sessions.save(session) + await self._publish(chat_id, waiting) + + async def _queue_continuation( + self, + session: Session, + chat_id: str, + state: Mapping[str, Any], + ) -> None: + recovery_id = cast(str, state["recovery_id"]) + await self.bus.publish_inbound( + InboundMessage( + channel="websocket", + sender_id="system:recovery", + chat_id=chat_id, + content=( + "Continue the interrupted request from the saved conversation context. " + "Do not repeat completed work or mention the restart unless it affects the answer." + ), + metadata={ + "webui": True, + "_wants_stream": True, + WEBUI_TURN_METADATA_KEY: f"recovery:{recovery_id}", + RECOVERY_INBOUND_METADATA_KEY: recovery_id, + turn_continuation.INTERNAL_CONTINUATION_META: True, + turn_continuation.SKIP_USER_PERSIST_META: True, + }, + session_key_override=session.key, + require_existing_session=True, + ) + ) + + async def _requeue_pending_followups(self, session: Session) -> None: + """Return durable live-turn follow-ups to the bus after a restart.""" + for message in pending_followups(session): + await self.bus.publish_inbound(message) + + @staticmethod + def _resume_message_count(session: Session) -> int | None: + raw_value = cast(object, session.metadata.get(RECOVERY_METADATA_KEY)) + value = cast(dict[str, Any], raw_value) if isinstance(raw_value, dict) else None + if value is None: + return None + count = value.get("resume_message_count") + return count if isinstance(count, int) and count >= 0 else None + + async def _publish( + self, + chat_id: str, + state: Mapping[str, Any], + ) -> None: + """Publish the recovery state and invalidate its sidebar projection.""" + await self.bus.publish_outbound( + outbound_message_for_event( + channel="websocket", + chat_id=chat_id, + event=RecoveryStateEvent( + status=cast(str, state["status"]), + recovery_id=cast(str, state["recovery_id"]), + reason=cast(str | None, state.get("reason")), + attempts=cast(int, state.get("attempts", 0)), + can_continue=cast(bool | None, state.get("can_continue")), + ), + ) + ) + await self.bus.publish_outbound( + outbound_message_for_event( + channel="websocket", + chat_id=chat_id, + event=SessionUpdatedEvent(scope="thread"), + ) + ) + + @staticmethod + def _set_state( + session: Session, + *, + status: str, + recovery_id: str, + attempts: int, + reason: str, + resume_message_count: int | None = None, + can_continue: bool = True, + ) -> dict[str, Any]: + state = { + "status": status, + "recovery_id": recovery_id, + "attempts": max(0, attempts), + "reason": reason, + "updated_at": datetime.now().isoformat(), + } + if not can_continue: + state["can_continue"] = False + if resume_message_count is not None: + state["resume_message_count"] = max(0, resume_message_count) + session.metadata[RECOVERY_METADATA_KEY] = state + session.updated_at = datetime.now() + return state + + def _session_key(self, chat_id: str) -> str: + return UNIFIED_SESSION_KEY if self.unified_session else f"websocket:{chat_id}" + + @staticmethod + def _has_unfinished_webui_transcript(session_key: str) -> bool: + """Detect a stale WebUI activity tail after an unclean gateway stop. + + The transcript is intentionally consulted only as a last-resort signal: + a durable pending turn or runtime checkpoint always takes precedence. + This keeps browser disconnects harmless while preventing a materialized + partial turn from being presented as active forever after a restart. + """ + try: + from nanobot.webui.transcript import has_unfinished_transcript_tail + + return has_unfinished_transcript_tail(session_key) + except (OSError, ValueError, TypeError): + # Recovery must fail closed if the optional display transcript is + # corrupt or unavailable; the normal checkpoint path still applies. + return False + + @staticmethod + def _websocket_route(session: Session) -> tuple[str, str] | None: + return RecoveryCoordinator._websocket_route_for(session.key, session.metadata) + + @staticmethod + def _websocket_route_for( + session_key: str, + metadata: Mapping[str, Any], + ) -> tuple[str, str] | None: + if session_key.startswith("websocket:"): + chat_id = session_key.split(":", 1)[1] + return ("websocket", chat_id) if chat_id else None + if session_key == UNIFIED_SESSION_KEY: + route = last_channel_from_metadata(metadata) + if route and route[0] == "websocket": + return route + return None diff --git a/nanobot/session/webui_turns.py b/nanobot/session/webui_turns.py index 8085f78ce..a6cf76b50 100644 --- a/nanobot/session/webui_turns.py +++ b/nanobot/session/webui_turns.py @@ -43,6 +43,7 @@ from nanobot.runtime_context import public_history_message from nanobot.session.goal_state import goal_state_ws_blob from nanobot.session.history_visibility import is_hidden_history_message from nanobot.session.manager import Session, SessionManager +from nanobot.session.recovery import RecoveryCoordinator from nanobot.session.session_handles import session_handle_for_name from nanobot.session.session_messages import ( SessionMessageEnvelope, @@ -511,6 +512,7 @@ class WebuiTurnCoordinator: bus: MessageBus sessions: SessionManager schedule_background: Callable[[Awaitable[None]], None] + recovery: RecoveryCoordinator | None = None def subscribe(self, runtime_events: RuntimeEventBus) -> Callable[[], None]: """Subscribe this coordinator to runtime events.""" @@ -654,6 +656,8 @@ class WebuiTurnCoordinator: event.runtime.context_window_tokens if event.runtime is not None else None ), ) + if self.recovery is not None: + await self.recovery.turn_completed(event.context.session_key) self._schedule_title_update_from_event(event) async def _handle_goal_state_changed(self, event: GoalStateChanged) -> None: diff --git a/nanobot/webui/gateway_services.py b/nanobot/webui/gateway_services.py index 68e4e94af..d9bf6b62d 100644 --- a/nanobot/webui/gateway_services.py +++ b/nanobot/webui/gateway_services.py @@ -69,6 +69,7 @@ def build_gateway_services( mcp_runtime_status: Callable[[], Mapping[str, str]] | None = None, mcp_reload: Callable[[], Awaitable[dict[str, Any]]] | None = None, skill_state_action: Callable[[set[str]], None] | None = None, + recovery_action: Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] | None = None, logger: Any = default_logger, ) -> GatewayServices: settings = WebUISettingsServices.create( @@ -131,6 +132,7 @@ def build_gateway_services( mcp_runtime_status=mcp_runtime_status, mcp_reload=mcp_reload, skill_state_action=skill_state_action, + recovery_action=recovery_action, log=logger, ) return GatewayServices( diff --git a/nanobot/webui/session_list_index.py b/nanobot/webui/session_list_index.py index 69f6c1aea..50612cfde 100644 --- a/nanobot/webui/session_list_index.py +++ b/nanobot/webui/session_list_index.py @@ -31,8 +31,9 @@ from nanobot.session.manager import ( _metadata_title, # pyright: ignore[reportPrivateUsage] ) from nanobot.session.model_selection import model_preset_from_metadata +from nanobot.session.recovery import recovery_state_from_metadata -_INDEX_VERSION = 7 +_INDEX_VERSION = 8 _INDEX_FILENAME = ".webui_session_index.json" _MODEL_PRESET_FIELD = "model_preset" _ROW_SOURCE_FIELD = "_source" @@ -245,6 +246,7 @@ def _public_row(sessions_dir: Path, webui_dir: Path, row: dict[str, Any]) -> dic "title": row.get("title", ""), "preview": row.get("preview", ""), _MODEL_PRESET_FIELD: row.get(_MODEL_PRESET_FIELD), + "recovery_state": row.get("recovery_state"), _WORKSPACE_SCOPE_PRESENT_FIELD: row.get(_WORKSPACE_SCOPE_PRESENT_FIELD, False), _WORKSPACE_SCOPE_VALUE_FIELD: row.get(_WORKSPACE_SCOPE_VALUE_FIELD), "path": str(path), @@ -485,6 +487,7 @@ def _indexed_row_for_session(session: Session, path: Path, webui_dir: Path) -> d "title": _metadata_title(session.metadata), "preview": _preview_from_messages(session.messages), _MODEL_PRESET_FIELD: model_preset_from_metadata(session.metadata), + "recovery_state": recovery_state_from_metadata(session.metadata), **_indexed_workspace_scope_fields(session.metadata), _ROW_SOURCE_FIELD: _SESSION_SOURCE, "file": path.name, @@ -601,6 +604,7 @@ def _scan_transcript_row( "title": "", "preview": preview or fallback_preview, _MODEL_PRESET_FIELD: None, + "recovery_state": None, **_indexed_workspace_scope_fields({}), _ROW_SOURCE_FIELD: _TRANSCRIPT_SOURCE, "file": stem, @@ -687,6 +691,7 @@ def _scan_session_row( "title": _metadata_title(metadata), "preview": preview or fallback_preview, _MODEL_PRESET_FIELD: model_preset_from_metadata(metadata), + "recovery_state": recovery_state_from_metadata(metadata), **_indexed_workspace_scope_fields(metadata), _ROW_SOURCE_FIELD: _SESSION_SOURCE, "file": path.name, diff --git a/nanobot/webui/transcript.py b/nanobot/webui/transcript.py index 80703c883..68ff0adfd 100644 --- a/nanobot/webui/transcript.py +++ b/nanobot/webui/transcript.py @@ -2541,6 +2541,19 @@ def has_pending_tool_calls( return False +def has_unfinished_transcript_tail(session_key: str) -> bool: + """Return whether the active transcript ends in an unfinished turn. + + Recovery runs at gateway startup and only needs the newest, still-active + turn. Completed turns are rotated into immutable segment files, so reading + every historical segment here would make restart cost grow with the full + conversation history. + """ + return has_pending_tool_calls( + _read_transcript_file(webui_transcript_path(session_key)) + ) + + def completed_turn_ids(lines: list[dict[str, Any]]) -> list[str]: """Return stable identities for turns with an explicitly persisted completion.""" completed: list[str] = [] diff --git a/nanobot/webui/ws_http.py b/nanobot/webui/ws_http.py index d6ac6d88d..eb64e778b 100644 --- a/nanobot/webui/ws_http.py +++ b/nanobot/webui/ws_http.py @@ -29,6 +29,7 @@ from nanobot.cron.session_turns import is_bound_cron_job from nanobot.cron.types import CronJob, CronSchedule from nanobot.security.workspace_access import WorkspaceScope from nanobot.session.manager import SessionManager +from nanobot.session.recovery import RecoveryActionError from nanobot.session.session_handles import ( SessionHandleResolver, ) @@ -145,6 +146,8 @@ _WEBUI_MUTATION_PATHS = { "skill.delete": "/api/webui/skills/delete", "sidebar.update": "/api/webui/sidebar-state/update", "workspace.pick_folder": "/api/workspaces/pick-folder", + "recovery.continue": "/api/webui/recovery/continue", + "recovery.dismiss": "/api/webui/recovery/dismiss", "settings.agent.update": "/api/settings/update", "settings.model_configuration.create": "/api/settings/model-configurations/create", "settings.model_configuration.update": "/api/settings/model-configurations/update", @@ -323,6 +326,9 @@ class GatewayHTTPHandler: mcp_runtime_status: Callable[[], Mapping[str, str]] | None = None, mcp_reload: Callable[[], Awaitable[dict[str, Any]]] | None = None, skill_state_action: Callable[[set[str]], None] | None = None, + recovery_action: ( + Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] | None + ) = None, log: Any = logger, ) -> None: self.config = config @@ -340,6 +346,7 @@ class GatewayHTTPHandler: disabled_skills if disabled_skills is not None else set() ) self.skill_state_action = skill_state_action + self.recovery_action = recovery_action self._skill_install_lock = asyncio.Lock() self._folder_picker_lock = asyncio.Lock() self.cron_service = cron_service @@ -454,6 +461,8 @@ class GatewayHTTPHandler: return True if re.match(r"^/api/webui/automations/(enable|disable|delete|run|update)$", path): return True + if path in {"/api/webui/recovery/continue", "/api/webui/recovery/dismiss"}: + return True return path in { "/api/webui/skills/install", "/api/webui/skills/update", @@ -507,6 +516,11 @@ class GatewayHTTPHandler: if response is not None: return response + # Recovery routes + response = await self._dispatch_recovery_route(request, got) + if response is not None: + return response + # Session routes response = await self._dispatch_session_routes(request, got) if response is not None: @@ -700,6 +714,27 @@ class GatewayHTTPHandler: return None + async def _dispatch_recovery_route( + self, + request: WsRequest, + path: str, + ) -> Response | None: + match = re.fullmatch(r"/api/webui/recovery/(continue|dismiss)", path) + if match is None: + return None + if not getattr(request, _WEBUI_MUTATION_REQUEST_ATTR, False): + return _http_error(405, "WebUI recovery actions require an authenticated WebSocket") + if self.recovery_action is None: + return _http_error(503, "WebUI recovery is unavailable") + payload = _mutation_payload(request) + if payload is None: + return _http_error(400, "invalid recovery payload") + try: + result = await self.recovery_action(match.group(1), payload) + except RecoveryActionError as exc: + return _http_error(exc.status, str(exc)) + return _http_json_response(result) + async def _handle_session_context_get(self, request: WsRequest, key: str) -> Response: if not self.check_api_token(request): return _http_error(401, "Unauthorized") @@ -746,6 +781,10 @@ class GatewayHTTPHandler: for k, v in s.items() if k != "path" and k not in WEBUI_SESSION_INDEX_INTERNAL_FIELDS } + # Keep the additive recovery field absent for ordinary sessions so + # older clients and compact list responses stay unchanged. + if row.get("recovery_state") is None: + row.pop("recovery_state", None) chat_id = key.split(":", 1)[1] started_at = websocket_turn_wall_started_at(chat_id) if started_at is not None: diff --git a/tests/agent/test_loop_save_turn.py b/tests/agent/test_loop_save_turn.py index 9b2ef8c96..25a96a8ce 100644 --- a/tests/agent/test_loop_save_turn.py +++ b/tests/agent/test_loop_save_turn.py @@ -36,6 +36,7 @@ from nanobot.session.keys import ( UNIFIED_SESSION_KEY, ) from nanobot.session.manager import Session +from nanobot.session.recovery import PENDING_FOLLOWUP_ID_KEY, PENDING_FOLLOWUPS_KEY from nanobot.session.turn_continuation import ( INTERNAL_CONTINUATION_META, INTERNAL_CONTINUATION_RUN_STARTED_AT_META, @@ -161,6 +162,35 @@ def test_persist_cron_turn_uses_distinct_history_marker(tmp_path: Path) -> None: assert message["cron_prompt_ref"] == prompt_ref +def test_persist_user_message_acknowledges_durable_followup(tmp_path: Path) -> None: + loop = _make_full_loop(tmp_path) + session = loop.sessions.get_or_create("websocket:chat") + session.metadata[PENDING_FOLLOWUPS_KEY] = [ + { + "id": "followup-1", + "sender_id": "user", + "chat_id": "chat", + "content": "queued while busy", + "media": [], + "metadata": {}, + } + ] + + persisted = loop._persist_user_message_early( + InboundMessage( + channel="websocket", + sender_id="user", + chat_id="chat", + content="queued while busy", + metadata={PENDING_FOLLOWUP_ID_KEY: "followup-1"}, + ), + session, + ) + + assert persisted is True + assert PENDING_FOLLOWUPS_KEY not in session.metadata + + def test_persist_local_trigger_turn_uses_hidden_automation_marker(tmp_path: Path) -> None: loop = _make_full_loop(tmp_path) session = loop.sessions.get_or_create("websocket:auto") @@ -381,6 +411,34 @@ def test_save_turn_keeps_multimodal_runtime_context_for_model_replay() -> None: assert public_history_message(session.messages[0])["content"] == [] +def test_save_turn_acknowledges_every_merged_recovery_followup() -> None: + """Persisting a merged injected row retires every durable follow-up ID.""" + loop = _mk_loop() + session = Session( + key="test:recovery-followups", + metadata={ + PENDING_FOLLOWUPS_KEY: [ + {"id": "first"}, + {"id": "second"}, + ] + }, + ) + + loop._save_turn( + session, + [ + { + "role": "user", + "content": "first\n\nsecond", + PENDING_FOLLOWUP_ID_KEY: ["first", "second"], + } + ], + skip=0, + ) + + assert PENDING_FOLLOWUPS_KEY not in session.metadata + + def test_save_turn_keeps_image_placeholder_and_runtime_context() -> None: loop = _mk_loop() session = Session(key="test:image") diff --git a/tests/agent/test_runner_injections.py b/tests/agent/test_runner_injections.py index 5fbed09b4..f4fb7f757 100644 --- a/tests/agent/test_runner_injections.py +++ b/tests/agent/test_runner_injections.py @@ -26,7 +26,7 @@ def _make_injection_callback(queue: asyncio.Queue): return inject_cb -def _make_loop(tmp_path): +def _make_loop(tmp_path, *, recovery_admission=None): from nanobot.agent.loop import AgentLoop from nanobot.bus.queue import MessageBus @@ -39,7 +39,12 @@ def _make_loop(tmp_path): patch("nanobot.agent.loop.SubagentManager") as mock_sub_mgr: mock_sub_mgr.return_value.cancel_by_session = AsyncMock(return_value=0) mock_sub_mgr.return_value.close = AsyncMock() - loop = AgentLoop(bus=bus, provider=provider, workspace=tmp_path) + loop = AgentLoop( + bus=bus, + provider=provider, + workspace=tmp_path, + recovery_admission=recovery_admission, + ) return loop @pytest.mark.asyncio @@ -759,6 +764,20 @@ async def test_runner_merges_multiple_injected_user_messages_without_losing_medi ) +def test_runner_merge_keeps_all_recovery_followup_ids() -> None: + """Merged follow-ups stay acknowledged together after a later save.""" + from nanobot.agent.runner import AgentRunner + from nanobot.session.recovery import PENDING_FOLLOWUP_ID_KEY + + messages = [{"role": "user", "content": "first", PENDING_FOLLOWUP_ID_KEY: "one"}] + AgentRunner._append_injected_messages( + messages, + [{"role": "user", "content": "second", PENDING_FOLLOWUP_ID_KEY: "two"}], + ) + + assert messages[-1][PENDING_FOLLOWUP_ID_KEY] == ["one", "two"] + + def test_runner_merge_preserves_runtime_markers_with_media() -> None: from nanobot.agent.runner import AgentRunner from nanobot.runtime_context import ( @@ -967,6 +986,38 @@ async def test_followup_routed_to_pending_queue(tmp_path): assert queued_msg.session_key == UNIFIED_SESSION_KEY +@pytest.mark.asyncio +async def test_websocket_followup_is_admitted_before_recovery_queue(tmp_path): + """Recovery admission runs before a newer WebUI message is injected.""" + from nanobot.bus.events import InboundMessage + + admission = MagicMock() + admission.admit = AsyncMock(return_value=True) + loop = _make_loop(tmp_path, recovery_admission=admission) + loop._dispatch = AsyncMock() # type: ignore[method-assign] + + session_key = "websocket:chat" + pending = asyncio.Queue(maxsize=20) + loop._pending_queues[session_key] = pending + + run_task = asyncio.create_task(loop.run()) + msg = InboundMessage( + channel="websocket", + sender_id="u", + chat_id="chat", + content="new request", + ) + await loop.bus.publish_inbound(msg) + + queued_msg = await asyncio.wait_for(pending.get(), timeout=2) + admission.admit.assert_awaited_once_with(msg) + assert queued_msg.content == msg.content + assert queued_msg.metadata["_recovery_followup_id"] + + loop.stop() + await asyncio.wait_for(run_task, timeout=2) + + @pytest.mark.asyncio async def test_mid_turn_subagent_result_does_not_resolve_a_new_turn_route(tmp_path): """Injected results stay inside the active turn instead of opening a side turn.""" @@ -1314,6 +1365,51 @@ async def test_pending_queue_full_falls_back_to_queued_task(tmp_path): assert pending.qsize() == 1 +@pytest.mark.asyncio +async def test_pending_queue_overflow_keeps_websocket_followup_durable(tmp_path): + """Fallback dispatch must not acknowledge a WebUI message before it commits.""" + from nanobot.bus.events import InboundMessage + from nanobot.session.manager import Session + from nanobot.session.recovery import pending_followups + + loop = _make_loop(tmp_path) + dispatched = asyncio.Event() + release_dispatch = asyncio.Event() + + async def _dispatch(_msg): + dispatched.set() + await release_dispatch.wait() + + loop._dispatch = AsyncMock(side_effect=_dispatch) # type: ignore[method-assign] + session = Session(key="websocket:c") + loop.sessions.get_or_create.return_value = session + pending = asyncio.Queue(maxsize=1) + pending.put_nowait( + InboundMessage(channel="websocket", sender_id="u", chat_id="c", content="already queued") + ) + loop._pending_queues["websocket:c"] = pending + + run_task = asyncio.create_task(loop.run()) + await loop.bus.publish_inbound( + InboundMessage( + channel="websocket", + sender_id="u", + chat_id="c", + content="durable follow-up", + metadata={"webui": True}, + ) + ) + await asyncio.wait_for(dispatched.wait(), timeout=2) + + assert [message.content for message in pending_followups(session)] == ["durable follow-up"] + dispatched_msg = loop._dispatch.await_args.args[0] + assert dispatched_msg.metadata["_recovery_followup_id"] + + release_dispatch.set() + loop.stop() + await asyncio.wait_for(run_task, timeout=2) + + @pytest.mark.asyncio async def test_dispatch_republishes_leftover_queue_messages(tmp_path): """Messages left in the pending queue after _dispatch are re-published to the bus. diff --git a/tests/agent/test_stop_preserves_context.py b/tests/agent/test_stop_preserves_context.py index e38c88a7c..52e26e493 100644 --- a/tests/agent/test_stop_preserves_context.py +++ b/tests/agent/test_stop_preserves_context.py @@ -161,3 +161,25 @@ async def test_dispatch_cancellation_restores_checkpoint(): "Checkpoint metadata should be cleared after restore" assert loop.sessions.save.called, \ "Session should be persisted so the restored state survives process restart" + + +@pytest.mark.asyncio +async def test_dispatch_cancellation_keeps_checkpoint_for_managed_restart(tmp_path: Path) -> None: + """A restart preserves the checkpoint; an explicit stop still restores it.""" + loop = _make_loop(tmp_path) + loop.preserve_inflight_turns_on_shutdown() + loop._restore_runtime_checkpoint = MagicMock() # type: ignore[method-assign] + + async def _cancel(*_args: object, **_kwargs: object) -> None: + raise asyncio.CancelledError() + + loop._process_message = _cancel # type: ignore[method-assign] + + from nanobot.bus.events import InboundMessage + + with pytest.raises(asyncio.CancelledError): + await loop._dispatch( + InboundMessage(channel="test", sender_id="u1", chat_id="c1", content="work") + ) + + loop._restore_runtime_checkpoint.assert_not_called() diff --git a/tests/channels/test_channel_plugins.py b/tests/channels/test_channel_plugins.py index c75b72e26..a223c37bb 100644 --- a/tests/channels/test_channel_plugins.py +++ b/tests/channels/test_channel_plugins.py @@ -3721,6 +3721,27 @@ async def test_notify_restart_done_waits_until_channel_starts(): assert sent_msg.content.startswith("Restart completed") +@pytest.mark.asyncio +async def test_websocket_restart_notice_does_not_overwrite_recovery_state(): + """WebSocket attach/recovery events already own reconnect state.""" + fake_config = SimpleNamespace( + channels=ChannelsConfig(), + providers=SimpleNamespace(groq=SimpleNamespace(api_key="")), + ) + mgr = ChannelManager.__new__(ChannelManager) + mgr.config = fake_config + mgr.bus = MessageBus() + channel = _StartableChannel(fake_config, mgr.bus) + channel._running = True + mgr.channels = {"websocket": channel} + mgr._send_with_retry = AsyncMock() + + notice = RestartNotice(channel="websocket", chat_id="chat", started_at_raw="100.0") + await mgr._send_restart_notice_when_started(notice) + + mgr._send_with_retry.assert_not_awaited() + + @pytest.mark.asyncio async def test_restart_notice_retries_until_running_channel_accepts_delivery(): """A running flag must not make an early transport failure final.""" diff --git a/tests/cli/test_commands.py b/tests/cli/test_commands.py index 2fc67aa7a..719e38fe8 100644 --- a/tests/cli/test_commands.py +++ b/tests/cli/test_commands.py @@ -103,6 +103,16 @@ class _GatewayAgentContractStub: return None +class _EmptyGatewaySessionManager: + """Minimal session-manager contract for gateway assembly tests.""" + + def list_sessions(self) -> list[dict[str, object]]: + return [] + + def flush_all(self) -> int: + return 0 + + def test_gateway_signal_handler_first_signal_stops_and_second_forces() -> None: class _FakeLoop: def __init__(self) -> None: @@ -2756,7 +2766,7 @@ def _patch_serve_runtime(monkeypatch, config: Config, seen: dict[str, object]) - monkeypatch, config, message_bus=lambda: object(), - session_manager=lambda _workspace: object(), + session_manager=lambda _workspace: _EmptyGatewaySessionManager(), ) monkeypatch.setattr("nanobot.cli.commands.AgentLoop", _FakeAgentLoop) monkeypatch.setattr("nanobot.api.server.create_app", _fake_create_app) @@ -2823,7 +2833,7 @@ def test_gateway_uses_workspace_directory_for_cron_store(monkeypatch, tmp_path: monkeypatch, config, message_bus=lambda: object(), - session_manager=lambda _workspace: object(), + session_manager=lambda _workspace: _EmptyGatewaySessionManager(), cron_service=_StopCron, ) @@ -3329,7 +3339,7 @@ def test_gateway_workspace_override_does_not_migrate_legacy_cron( monkeypatch, config, message_bus=lambda: object(), - session_manager=lambda _workspace: object(), + session_manager=lambda _workspace: _EmptyGatewaySessionManager(), cron_service=_StopCron, get_cron_dir=lambda: legacy_dir, ) @@ -3368,7 +3378,7 @@ def test_gateway_custom_config_workspace_does_not_migrate_legacy_cron( monkeypatch, config, message_bus=lambda: object(), - session_manager=lambda _workspace: object(), + session_manager=lambda _workspace: _EmptyGatewaySessionManager(), cron_service=_StopCron, get_cron_dir=lambda: legacy_dir, ) @@ -3569,7 +3579,7 @@ def test_gateway_health_endpoint_binds_and_serves_expected_responses( monkeypatch, config, message_bus=lambda: object(), - session_manager=lambda _workspace: object(), + session_manager=lambda _workspace: _EmptyGatewaySessionManager(), ) monkeypatch.setattr("nanobot.cli.gateway_runtime.AgentLoop", _FakeAgentLoop) monkeypatch.setattr("nanobot.channels.manager.ChannelManager", _FakeChannelManager) @@ -3771,7 +3781,7 @@ def test_gateway_agent_task_owns_initial_mcp_provider_close( monkeypatch, config, message_bus=lambda: object(), - session_manager=lambda _workspace: object(), + session_manager=lambda _workspace: _EmptyGatewaySessionManager(), ) monkeypatch.setattr("nanobot.cli.gateway_runtime.AgentLoop", _FakeAgentLoop) monkeypatch.setattr("nanobot.cli.gateway_runtime.MCPProvider", _FakeMCPProvider) @@ -3894,7 +3904,7 @@ def test_gateway_shutdown_event_exits_forever_runtime_tasks( monkeypatch, config, message_bus=lambda: object(), - session_manager=lambda _workspace: object(), + session_manager=lambda _workspace: _EmptyGatewaySessionManager(), ) monkeypatch.setattr("nanobot.cli.gateway_runtime.AgentLoop", _FakeAgentLoop) monkeypatch.setattr("nanobot.channels.manager.ChannelManager", _FakeChannelManager) diff --git a/tests/gateway/test_runtime.py b/tests/gateway/test_runtime.py index 7c0984eea..6bf603fdc 100644 --- a/tests/gateway/test_runtime.py +++ b/tests/gateway/test_runtime.py @@ -18,6 +18,7 @@ from nanobot.gateway import ( GatewayRuntimePaths, GatewayStartOptions, GatewayStatus, + RuntimeResult, ) from nanobot.gateway.runtime import monitor_gateway_clients from nanobot.process_runtime import process_is_running @@ -454,6 +455,48 @@ def test_restart_does_not_detach_a_foreground_gateway(tmp_path, monkeypatch): assert result.message == "gateway_foreground_restart_required" +def test_restart_marks_the_exiting_gateway_for_turn_recovery(tmp_path, monkeypatch): + """A managed restart must not look like an explicit stop to the child.""" + runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux") + status = GatewayStatus( + running=True, + pid=12345, + state_path=runtime.paths.state_path, + log_path=runtime.paths.log_path, + launch_mode="background", + ) + monkeypatch.setattr(runtime, "status", lambda **_kwargs: status) + + def stop(*, timeout_s: int): + assert timeout_s == 20 + assert json.loads(runtime._restart_intent_path.read_text(encoding="utf-8")) == { + "pid": 12345 + } + return SimpleNamespace(ok=True, message="gateway_stopped", status=status) + + monkeypatch.setattr(runtime, "_stop", stop) + monkeypatch.setattr( + runtime, + "_start_background", + lambda _options: RuntimeResult(True, "gateway_started_background", status), + ) + + result = runtime.restart(GatewayStartOptions(port=18790)) + + assert result.ok is True + assert not runtime._restart_intent_path.exists() + + +def test_restart_intent_only_applies_to_the_recorded_gateway_process(tmp_path): + runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux") + + runtime._write_restart_intent(os.getpid()) + + assert runtime.preserves_inflight_turns_on_exit() is True + runtime._write_restart_intent(os.getpid() + 1) + assert runtime.preserves_inflight_turns_on_exit() is False + + def test_last_interactive_client_stops_an_on_demand_gateway(tmp_path, monkeypatch): runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux") monkeypatch.setattr(runtime, "_process_identity", lambda pid: pid) diff --git a/tests/session/test_recovery.py b/tests/session/test_recovery.py new file mode 100644 index 000000000..7175c7109 --- /dev/null +++ b/tests/session/test_recovery.py @@ -0,0 +1,732 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from nanobot.bus.events import InboundMessage +from nanobot.bus.outbound_events import RecoveryStateEvent, SessionUpdatedEvent +from nanobot.bus.queue import MessageBus +from nanobot.session.manager import Session, SessionManager +from nanobot.session.recovery import ( + PENDING_FOLLOWUPS_KEY, + PENDING_USER_TURN_KEY, + RECOVERY_METADATA_KEY, + RUNTIME_CHECKPOINT_KEY, + RecoveryActionError, + RecoveryCoordinator, + acknowledge_pending_followups, + pending_followups, + record_pending_followup, +) +from nanobot.webui import session_list_index, transcript + + +def _persist(manager: SessionManager, session: Session) -> None: + session.metadata["webui"] = True + manager.save(session) + + +def _coordinator(workspace: Path) -> tuple[RecoveryCoordinator, MessageBus, SessionManager]: + bus = MessageBus() + sessions = SessionManager(workspace) + return RecoveryCoordinator(sessions, bus), bus, sessions + + +@pytest.mark.asyncio +async def test_restart_before_model_call_waits_for_confirmation(tmp_path: Path) -> None: + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "finish this"}) + session.metadata[PENDING_USER_TURN_KEY] = True + _persist(sessions, session) + + coordinator, bus, restarted = _coordinator(tmp_path) + await coordinator.scan() + + assert bus.inbound.empty() + restored = restarted.get_or_create("websocket:chat") + assert restored.metadata[PENDING_USER_TURN_KEY] is True + assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user" + assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "restart_requires_confirmation" + + +@pytest.mark.asyncio +async def test_stale_incomplete_transcript_waits_for_confirmation(tmp_path: Path, monkeypatch) -> None: + """A materialized shutdown must not reappear as an endless Working state.""" + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + _persist(sessions, session) + monkeypatch.setattr( + "nanobot.webui.transcript.has_unfinished_transcript_tail", + lambda _key: True, + ) + + coordinator, bus, restarted = _coordinator(tmp_path) + await coordinator.scan() + + assert bus.inbound.empty() + state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY] + assert state["status"] == "awaiting_user" + assert state["reason"] == "interrupted_without_checkpoint" + event = bus.outbound.get_nowait().event + assert isinstance(event, RecoveryStateEvent) + assert event.status == "awaiting_user" + + +@pytest.mark.asyncio +async def test_transcript_only_interruption_is_discovered_without_materializing_completed_history( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + webui_dir = tmp_path / "webui" + webui_dir.mkdir() + monkeypatch.setattr(session_list_index, "get_webui_dir", lambda: webui_dir) + monkeypatch.setattr(transcript, "get_webui_dir", lambda: webui_dir) + unfinished_key = "websocket:unfinished" + completed_key = "websocket:completed" + (webui_dir / f"{SessionManager.safe_key(unfinished_key)}.jsonl").write_text( + '{"event":"user","chat_id":"unfinished","text":"keep going"}\n' + '{"event":"message","chat_id":"unfinished","kind":"progress","text":"Working"}\n', + encoding="utf-8", + ) + (webui_dir / f"{SessionManager.safe_key(completed_key)}.jsonl").write_text( + '{"event":"user","chat_id":"completed","text":"done"}\n' + '{"event":"message","chat_id":"completed","text":"finished"}\n' + '{"event":"turn_end","chat_id":"completed"}\n', + encoding="utf-8", + ) + coordinator, bus, sessions = _coordinator(tmp_path / "workspace") + + await coordinator.scan() + + restored = sessions.get_or_create(unfinished_key) + assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user" + assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "interrupted_without_checkpoint" + assert restored.metadata[RECOVERY_METADATA_KEY]["can_continue"] is False + assert sessions.read_session_metadata(completed_key) is None + event = bus.outbound.get_nowait().event + assert isinstance(event, RecoveryStateEvent) + assert event.status == "awaiting_user" + assert event.can_continue is False + assert bus.outbound.get_nowait().event.scope == "thread" + assert bus.outbound.empty() + + with pytest.raises(RecoveryActionError, match="context is unavailable"): + await coordinator.handle_action( + "continue", + {"chat_id": "unfinished", "recovery_id": event.recovery_id}, + ) + + +@pytest.mark.asyncio +async def test_scan_loads_only_sessions_that_need_webui_recovery( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sessions = SessionManager(tmp_path) + for key in ("telegram:idle", "discord:pending", "websocket:idle"): + session = sessions.get_or_create(key) + if key == "discord:pending": + session.metadata[PENDING_USER_TURN_KEY] = True + _persist(sessions, session) + pending = sessions.get_or_create("websocket:pending") + pending.metadata[PENDING_USER_TURN_KEY] = True + _persist(sessions, pending) + + coordinator, _, restarted = _coordinator(tmp_path) + loaded: list[str] = [] + get_or_create = restarted.get_or_create + + def tracked_get_or_create(key: str) -> Session: + loaded.append(key) + return get_or_create(key) + + monkeypatch.setattr(restarted, "get_or_create", tracked_get_or_create) + monkeypatch.setattr( + "nanobot.webui.transcript.has_unfinished_transcript_tail", + lambda _key: False, + ) + + await coordinator.scan() + + assert loaded == ["websocket:pending"] + + +@pytest.mark.asyncio +async def test_live_turn_followup_survives_restart_until_it_is_committed(tmp_path: Path) -> None: + """A message injected mid-turn is not lost between checkpoints.""" + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + _persist(sessions, session) + followup_id = record_pending_followup( + session, + InboundMessage( + channel="websocket", + sender_id="user", + chat_id="chat", + content="also check the logs", + metadata={"webui": True}, + ), + ) + assert followup_id is not None + sessions.save(session) + + coordinator, bus, restarted = _coordinator(tmp_path) + await coordinator.scan() + + queued = bus.inbound.get_nowait() + assert queued.content == "also check the logs" + assert queued.metadata["_recovery_followup_id"] == followup_id + restored = restarted.get_or_create("websocket:chat") + assert len(pending_followups(restored)) == 1 + + acknowledge_pending_followups(restored, [followup_id]) + assert PENDING_FOLLOWUPS_KEY not in restored.metadata + + +def test_followup_journal_keeps_every_uncommitted_message(tmp_path: Path) -> None: + """A live queue limit must never truncate durable WebUI follow-ups.""" + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + followup_ids = [ + record_pending_followup( + session, + InboundMessage( + channel="websocket", + sender_id="user", + chat_id="chat", + content=f"follow-up-{index}", + metadata={"webui": True}, + ), + ) + for index in range(21) + ] + + assert all(followup_ids) + sessions.save(session) + restarted = SessionManager(tmp_path) + restored = restarted.get_or_create("websocket:chat") + assert [message.content for message in pending_followups(restored)] == [ + f"follow-up-{index}" for index in range(21) + ] + + +def test_requeued_followup_preserves_its_journal_id(tmp_path: Path) -> None: + """Routing a recovered follow-up into a live turn must remain idempotent.""" + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + original_id = record_pending_followup( + session, + InboundMessage( + channel="websocket", + sender_id="user", + chat_id="chat", + content="also check the logs", + metadata={"webui": True}, + ), + ) + assert original_id is not None + + recovered = pending_followups(session)[0] + assert record_pending_followup(session, recovered) == original_id + assert [record["id"] for record in session.metadata[PENDING_FOLLOWUPS_KEY]] == [ + original_id + ] + + +@pytest.mark.asyncio +async def test_completed_tools_wait_for_confirmation_after_restart(tmp_path: Path) -> None: + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "inspect"}) + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = { + "phase": "tools_completed", + "assistant_message": { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "call-1", "function": {"name": "read_file"}}], + }, + "completed_tool_results": [ + { + "role": "tool", + "tool_call_id": "call-1", + "name": "read_file", + "content": "saved result", + } + ], + "pending_tool_calls": [], + } + _persist(sessions, session) + + coordinator, bus, restarted = _coordinator(tmp_path) + await coordinator.scan() + + assert bus.inbound.empty() + restored = restarted.get_or_create("websocket:chat") + assert restored.messages[-1]["content"] == "saved result" + assert RUNTIME_CHECKPOINT_KEY not in restored.metadata + assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user" + + +@pytest.mark.asyncio +async def test_uncertain_tool_is_never_replayed(tmp_path: Path) -> None: + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "send it"}) + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = { + "phase": "awaiting_tools", + "assistant_message": { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "call-1", "function": {"name": "send_email"}}], + }, + "completed_tool_results": [], + "pending_tool_calls": [ + {"id": "call-1", "function": {"name": "send_email"}} + ], + } + _persist(sessions, session) + + coordinator, bus, restarted = _coordinator(tmp_path) + await coordinator.scan() + + assert bus.inbound.empty() + restored = restarted.get_or_create("websocket:chat") + assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user" + assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "tool_state_unknown" + assert restored.messages[-1]["_recovery_interrupted"] is True + event = bus.outbound.get_nowait().event + assert isinstance(event, RecoveryStateEvent) + assert event.status == "awaiting_user" + + +@pytest.mark.asyncio +async def test_unknown_checkpoint_waits_for_confirmation(tmp_path: Path) -> None: + """Malformed or newer checkpoint phases fail closed pending confirmation.""" + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "deploy it"}) + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = {"phase": "future_phase"} + _persist(sessions, session) + + coordinator, bus, restarted = _coordinator(tmp_path) + await coordinator.scan() + + assert bus.inbound.empty() + restored = restarted.get_or_create("websocket:chat") + assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user" + assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "checkpoint_unknown" + assert restored.metadata[RECOVERY_METADATA_KEY]["can_continue"] is False + assert RUNTIME_CHECKPOINT_KEY not in restored.metadata + assert PENDING_USER_TURN_KEY not in restored.metadata + assert [message["role"] for message in restored.messages] == ["user", "assistant"] + assert restored.messages[-1]["_recovery_interrupted"] is True + + with pytest.raises(RecoveryActionError, match="context is unavailable"): + await coordinator.handle_action( + "continue", + { + "chat_id": "chat", + "recovery_id": restored.metadata[RECOVERY_METADATA_KEY]["recovery_id"], + }, + ) + + dismissed = await coordinator.handle_action( + "dismiss", + { + "chat_id": "chat", + "recovery_id": restored.metadata[RECOVERY_METADATA_KEY]["recovery_id"], + }, + ) + assert dismissed["status"] == "recovered" + + +@pytest.mark.asyncio +async def test_malformed_checkpoint_can_always_be_dismissed(tmp_path: Path) -> None: + """Corrupt private state must not trap the user in a failed recovery notice.""" + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "deploy it"}) + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = { + "phase": "future_phase", + "assistant_message": "invalid", + "completed_tool_results": 3, + "pending_tool_calls": [{"id": "call-1", "function": "invalid"}], + } + _persist(sessions, session) + + coordinator, _, restarted = _coordinator(tmp_path) + await coordinator.scan() + restored = restarted.get_or_create("websocket:chat") + state = restored.metadata[RECOVERY_METADATA_KEY] + + result = await coordinator.handle_action( + "dismiss", + {"chat_id": "chat", "recovery_id": state["recovery_id"]}, + ) + + assert result["status"] == "recovered" + assert RUNTIME_CHECKPOINT_KEY not in restored.metadata + + +@pytest.mark.asyncio +async def test_known_but_malformed_checkpoint_cannot_continue(tmp_path: Path) -> None: + """A known phase does not make corrupt tool state safe to resume.""" + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "send it"}) + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = { + "phase": "tools_completed", + "assistant_message": {"role": "assistant", "content": "working"}, + "completed_tool_results": "missing durable results", + "pending_tool_calls": [], + } + _persist(sessions, session) + + coordinator, _, restarted = _coordinator(tmp_path) + await coordinator.scan() + restored = restarted.get_or_create("websocket:chat") + state = restored.metadata[RECOVERY_METADATA_KEY] + + assert state["status"] == "awaiting_user" + assert state["reason"] == "checkpoint_invalid" + assert state["can_continue"] is False + with pytest.raises(RecoveryActionError, match="context is unavailable"): + await coordinator.handle_action( + "continue", + {"chat_id": "chat", "recovery_id": state["recovery_id"]}, + ) + + +@pytest.mark.asyncio +async def test_malformed_final_response_is_not_reported_as_restored(tmp_path: Path) -> None: + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "answer"}) + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = { + "phase": "final_response", + "assistant_message": "not an answer row", + "completed_tool_results": [], + "pending_tool_calls": [], + } + _persist(sessions, session) + + coordinator, _, restarted = _coordinator(tmp_path) + await coordinator.scan() + + state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY] + assert state["status"] == "awaiting_user" + assert state["reason"] == "checkpoint_invalid" + assert state["can_continue"] is False + restored = restarted.get_or_create("websocket:chat") + assert RUNTIME_CHECKPOINT_KEY not in restored.metadata + assert PENDING_USER_TURN_KEY not in restored.metadata + assert [message["role"] for message in restored.messages] == ["user", "assistant"] + assert restored.messages[-1]["_recovery_interrupted"] is True + assert all("tool_calls" not in message for message in restored.messages) + + +@pytest.mark.asyncio +async def test_checkpoint_with_missing_tool_result_cannot_continue(tmp_path: Path) -> None: + """Never resume when persisted results do not cover every requested tool.""" + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "send both"}) + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = { + "phase": "tools_completed", + "assistant_message": { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call-1", "function": {"name": "send_email"}}, + {"id": "call-2", "function": {"name": "send_email"}}, + ], + }, + "completed_tool_results": [ + { + "role": "tool", + "tool_call_id": "call-1", + "name": "send_email", + "content": "sent", + } + ], + "pending_tool_calls": [], + } + _persist(sessions, session) + + coordinator, _, restarted = _coordinator(tmp_path) + await coordinator.scan() + + restored = restarted.get_or_create("websocket:chat") + state = restored.metadata[RECOVERY_METADATA_KEY] + assert state["status"] == "awaiting_user" + assert state["reason"] == "checkpoint_invalid" + assert state["can_continue"] is False + assert RUNTIME_CHECKPOINT_KEY not in restored.metadata + assert PENDING_USER_TURN_KEY not in restored.metadata + assert [message["role"] for message in restored.messages] == ["user", "assistant"] + assert restored.messages[-1]["_recovery_interrupted"] is True + assert all("tool_calls" not in message for message in restored.messages) + + +@pytest.mark.asyncio +async def test_empty_final_response_is_not_reported_as_restored(tmp_path: Path) -> None: + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "answer"}) + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = { + "phase": "final_response", + "assistant_message": {"role": "assistant", "content": ""}, + "completed_tool_results": [], + "pending_tool_calls": [], + } + _persist(sessions, session) + + coordinator, _, restarted = _coordinator(tmp_path) + await coordinator.scan() + + state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY] + assert state["status"] == "awaiting_user" + assert state["reason"] == "checkpoint_invalid" + assert state["can_continue"] is False + + +@pytest.mark.asyncio +async def test_final_answer_is_restored_without_model_call(tmp_path: Path) -> None: + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "answer"}) + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = { + "phase": "final_response", + "assistant_message": {"role": "assistant", "content": "already finished"}, + "completed_tool_results": [], + "pending_tool_calls": [], + } + _persist(sessions, session) + + coordinator, bus, restarted = _coordinator(tmp_path) + await coordinator.scan() + + assert bus.inbound.empty() + restored = restarted.get_or_create("websocket:chat") + assert restored.messages[-1]["content"] == "already finished" + first = bus.outbound.get_nowait().event + second = bus.outbound.get_nowait().event + assert isinstance(first, RecoveryStateEvent) and first.status == "recovered" + assert isinstance(second, SessionUpdatedEvent) + + +@pytest.mark.asyncio +async def test_explicit_recovery_continue_queues_once(tmp_path: Path) -> None: + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "continue"}) + session.metadata[PENDING_USER_TURN_KEY] = True + _persist(sessions, session) + + coordinator, bus, restarted = _coordinator(tmp_path) + await coordinator.scan() + assert bus.inbound.empty() + + state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY] + result = await coordinator.handle_action( + "continue", + {"chat_id": "chat", "recovery_id": state["recovery_id"]}, + ) + assert result["status"] == "resuming" + assert bus.inbound.get_nowait().metadata["_webui_recovery_id"] == state["recovery_id"] + + +@pytest.mark.asyncio +async def test_new_user_message_supersedes_waiting_recovery(tmp_path: Path) -> None: + coordinator, bus, sessions = _coordinator(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.append({"role": "user", "content": "old request"}) + session.metadata[PENDING_USER_TURN_KEY] = True + _persist(sessions, session) + await coordinator.scan() + + newer = InboundMessage( + channel="websocket", + sender_id="user", + chat_id="chat", + content="new request", + ) + assert await coordinator.admit(newer) is True + restored = sessions.get_or_create("websocket:chat") + assert restored.messages[-1]["_recovery_interrupted"] is True + assert sum( + message.get("_recovery_interrupted") is True + for message in restored.messages + ) == 1 + assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "superseded" + + +@pytest.mark.asyncio +async def test_new_user_message_cancels_active_recovery_task(tmp_path: Path) -> None: + coordinator, bus, sessions = _coordinator(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.metadata[RECOVERY_METADATA_KEY] = { + "status": "resuming", + "recovery_id": "active", + "attempts": 1, + } + _persist(sessions, session) + + started = asyncio.Event() + + async def _active_recovery() -> None: + started.set() + await asyncio.Event().wait() + + task = asyncio.create_task(_active_recovery()) + await started.wait() + coordinator.register_recovery_task("websocket:chat", task) + + newer = InboundMessage( + channel="websocket", + sender_id="user", + chat_id="chat", + content="new request", + ) + assert await coordinator.admit(newer) is True + assert task.cancelled() + restored = sessions.get_or_create("websocket:chat") + assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "superseded" + assert restored.messages[-1]["_recovery_interrupted"] is True + first = bus.outbound.get_nowait().event + second = bus.outbound.get_nowait().event + assert isinstance(first, RecoveryStateEvent) + assert isinstance(second, SessionUpdatedEvent) + assert bus.outbound.empty() + + +@pytest.mark.asyncio +async def test_recovery_action_rejects_stale_page_and_continues_current_state( + tmp_path: Path, +) -> None: + coordinator, bus, sessions = _coordinator(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.metadata[RECOVERY_METADATA_KEY] = { + "status": "awaiting_user", + "recovery_id": "current", + "attempts": 0, + } + _persist(sessions, session) + + with pytest.raises(RecoveryActionError, match="stale"): + await coordinator.handle_action( + "continue", + {"chat_id": "chat", "recovery_id": "old"}, + ) + + result = await coordinator.handle_action( + "continue", + {"chat_id": "chat", "recovery_id": "current"}, + ) + assert result["status"] == "resuming" + assert bus.inbound.get_nowait().metadata["_webui_recovery_id"] == "current" + + +@pytest.mark.asyncio +async def test_persisted_completion_wins_over_stale_resuming_marker(tmp_path: Path) -> None: + sessions = SessionManager(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.messages.extend( + [ + {"role": "user", "content": "work"}, + {"role": "assistant", "content": "done"}, + ] + ) + session.metadata[RECOVERY_METADATA_KEY] = { + "status": "resuming", + "recovery_id": "recovery", + "attempts": 1, + "resume_message_count": 1, + } + _persist(sessions, session) + + coordinator, bus, restarted = _coordinator(tmp_path) + await coordinator.scan() + + assert bus.inbound.empty() + state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY] + assert state["status"] == "recovered" + assert state["reason"] == "committed" + + +@pytest.mark.asyncio +async def test_dismiss_does_not_queue_work(tmp_path: Path) -> None: + coordinator, bus, sessions = _coordinator(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.metadata[RECOVERY_METADATA_KEY] = { + "status": "awaiting_user", + "recovery_id": "current", + "attempts": 0, + } + _persist(sessions, session) + + result = await coordinator.handle_action( + "dismiss", + {"chat_id": "chat", "recovery_id": "current"}, + ) + + assert result["status"] == "recovered" + assert bus.inbound.empty() + + +@pytest.mark.asyncio +async def test_scan_failure_is_visible_instead_of_aborting_other_sessions( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + coordinator, bus, sessions = _coordinator(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.metadata[PENDING_USER_TURN_KEY] = True + _persist(sessions, session) + + async def fail(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("boom") + + monkeypatch.setattr(RecoveryCoordinator, "_recover_session", fail) + await coordinator.scan() + + state = sessions.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY] + assert state["status"] == "failed" + assert state["can_continue"] is False + assert isinstance(bus.outbound.get_nowait().event, RecoveryStateEvent) + + with pytest.raises(RecoveryActionError, match="context is unavailable"): + await coordinator.handle_action( + "continue", + {"chat_id": "chat", "recovery_id": state["recovery_id"]}, + ) + + +@pytest.mark.asyncio +async def test_bus_remains_quiet_after_recovered_state(tmp_path: Path) -> None: + coordinator, bus, sessions = _coordinator(tmp_path) + session = sessions.get_or_create("websocket:chat") + session.metadata[RECOVERY_METADATA_KEY] = { + "status": "recovered", + "recovery_id": "done", + "attempts": 1, + } + _persist(sessions, session) + + await coordinator.scan() + + await asyncio.sleep(0) + assert bus.inbound.empty() + assert bus.outbound.empty() diff --git a/tests/session/test_session_store.py b/tests/session/test_session_store.py index 1c6d7c1ce..a64a9c626 100644 --- a/tests/session/test_session_store.py +++ b/tests/session/test_session_store.py @@ -1,6 +1,7 @@ from unittest.mock import MagicMock import nanobot.session as session_api +from nanobot.providers.base import ProviderConversationState from nanobot.session import Session, SessionManager from nanobot.session.manager import SessionStore from nanobot.session.model_selection import SESSION_MODEL_PRESET_METADATA_KEY @@ -123,3 +124,93 @@ def test_manager_preserves_full_session_before_store_save(tmp_path) -> None: assert session.messages[0]["content"] == "0" assert session.messages[-1]["content"] == "2000" store.save.assert_called_once_with(session, fsync=False) + + +def test_runtime_checkpoint_does_not_rewrite_long_session(tmp_path) -> None: + manager = SessionManager(tmp_path) + session = manager.get_or_create("websocket:long") + for index in range(256): + session.add_message("user", f"{index}:" + "x" * 4096) + manager.save(session) + + main_path = manager._get_session_path(session.key) + main_before = main_path.read_bytes() + stat_before = main_path.stat() + session.metadata["runtime_checkpoint"] = { + "phase": "tools_completed", + "assistant_message": {"role": "assistant", "content": "working"}, + "completed_tool_results": [], + "pending_tool_calls": [], + } + session.provider_state = ProviderConversationState( + kind="openai_responses", + provider="openai:test", + model="test-model", + version=1, + payload={"response_id": "private-response"}, + ) + + manager.save_runtime_checkpoint(session) + + checkpoint_path = manager._get_runtime_checkpoint_path(session.key) + assert main_path.read_bytes() == main_before + assert main_path.stat().st_ino == stat_before.st_ino + assert main_path.stat().st_mtime_ns == stat_before.st_mtime_ns + assert checkpoint_path.stat().st_size < len(main_before) // 100 + + restored = SessionManager(tmp_path).get_or_create(session.key) + assert restored.metadata["runtime_checkpoint"]["phase"] == "tools_completed" + assert restored.provider_state is not None + assert restored.provider_state.payload == {"response_id": "private-response"} + + +def test_completed_session_supersedes_stale_checkpoint(tmp_path) -> None: + manager = SessionManager(tmp_path) + session = manager.get_or_create("websocket:completed") + session.add_message("user", "question") + manager.save(session) + session.metadata["runtime_checkpoint"] = {"phase": "awaiting_tools"} + manager.save_runtime_checkpoint(session) + checkpoint_path = manager._get_runtime_checkpoint_path(session.key) + stale_checkpoint = checkpoint_path.read_bytes() + + session.metadata.pop("runtime_checkpoint") + session.add_message("assistant", "answer") + manager.save(session) + assert not checkpoint_path.exists() + + # Emulate a process dying after the main record was committed but before an + # obsolete sidecar could be removed. The base fingerprint keeps it stale. + checkpoint_path.write_bytes(stale_checkpoint) + restored = SessionManager(tmp_path).get_or_create(session.key) + assert "runtime_checkpoint" not in restored.metadata + assert restored.messages[-1]["content"] == "answer" + assert not checkpoint_path.exists() + + +def test_delete_session_removes_runtime_checkpoint(tmp_path) -> None: + manager = SessionManager(tmp_path) + session = manager.get_or_create("websocket:delete") + session.add_message("user", "question") + manager.save(session) + session.metadata["runtime_checkpoint"] = {"phase": "awaiting_tools"} + manager.save_runtime_checkpoint(session) + checkpoint_path = manager._get_runtime_checkpoint_path(session.key) + assert checkpoint_path.exists() + + assert manager.delete_session(session.key) is True + assert not checkpoint_path.exists() + + +def test_invalid_runtime_checkpoint_is_discarded(tmp_path) -> None: + manager = SessionManager(tmp_path) + session = manager.get_or_create("websocket:invalid-checkpoint") + session.add_message("user", "question") + manager.save(session) + checkpoint_path = manager._get_runtime_checkpoint_path(session.key) + checkpoint_path.write_text("{truncated", encoding="utf-8") + + restored = SessionManager(tmp_path).get_or_create(session.key) + + assert "runtime_checkpoint" not in restored.metadata + assert not checkpoint_path.exists() diff --git a/tests/utils/test_webui_transcript.py b/tests/utils/test_webui_transcript.py index 738fd83cd..bbd7d2157 100644 --- a/tests/utils/test_webui_transcript.py +++ b/tests/utils/test_webui_transcript.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + import nanobot.webui.transcript as transcript_module from nanobot.session.history_visibility import HIDDEN_HISTORY_META from nanobot.webui.transcript import ( @@ -554,6 +556,22 @@ def test_thread_response_marks_unfinished_tool_tail_pending(tmp_path, monkeypatc assert out["completed_turn_ids"] == [] +def test_recovery_tail_check_reads_only_the_active_transcript(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path) + key = "websocket:recovery-tail" + active_path = transcript_module.webui_transcript_path(key) + reads: list[Path] = [] + + def read(path: Path) -> list[dict[str, object]]: + reads.append(path) + return [{"event": "message", "kind": "progress", "text": "running"}] + + monkeypatch.setattr(transcript_module, "_read_transcript_file", read) + + assert transcript_module.has_unfinished_transcript_tail(key) is True + assert reads == [active_path] + + def test_thread_response_reports_active_registry_without_transcript( tmp_path, monkeypatch, diff --git a/tests/webui/test_gateway_webui_smoke.py b/tests/webui/test_gateway_webui_smoke.py index 535abb939..6b63f7c44 100644 --- a/tests/webui/test_gateway_webui_smoke.py +++ b/tests/webui/test_gateway_webui_smoke.py @@ -15,6 +15,9 @@ import httpx import pytest import websockets +from nanobot.session.manager import SessionManager +from nanobot.session.recovery import PENDING_USER_TURN_KEY, RUNTIME_CHECKPOINT_KEY + _BOOTSTRAP_SECRET = "smoke-secret" @@ -206,3 +209,84 @@ async def test_gateway_webui_bootstrap_message_and_thread_hydration(tmp_path: Pa assert any("shell-ok" in text for text in contents) finally: _stop_gateway(process) + + +def test_gateway_restart_restores_a_completed_answer_without_replaying_model( + tmp_path: Path, +) -> None: + """Exercise recovery through two real gateway processes and durable files.""" + ws_port = _free_port() + gateway_port = _free_port() + workspace = tmp_path / "workspace" + workspace.mkdir() + config_path = tmp_path / "config.json" + first_log = tmp_path / "gateway-first.log" + second_log = tmp_path / "gateway-second.log" + _write_smoke_config( + config_path, + workspace=workspace, + ws_port=ws_port, + gateway_port=gateway_port, + ) + base_url = f"http://127.0.0.1:{ws_port}" + + first = _start_gateway(config_path, first_log) + try: + _wait_for_bootstrap(base_url, first, first_log) + finally: + _stop_gateway(first) + + sessions_root = tmp_path / "sessions" + sessions = SessionManager(workspace, sessions_root=sessions_root) + session = sessions.get_or_create("websocket:recovery-smoke") + session.messages.append({"role": "user", "content": "recover this answer"}) + session.metadata["webui"] = True + session.metadata[PENDING_USER_TURN_KEY] = True + session.metadata[RUNTIME_CHECKPOINT_KEY] = { + "phase": "final_response", + "assistant_message": { + "role": "assistant", + "content": "restored without another model request", + }, + "completed_tool_results": [], + "pending_tool_calls": [], + } + sessions.save(session, fsync=True) + + second = _start_gateway(config_path, second_log) + try: + bootstrap = _wait_for_bootstrap(base_url, second, second_log) + deadline = time.monotonic() + 20 + restored = None + while time.monotonic() < deadline: + restored = SessionManager( + workspace, + sessions_root=sessions_root, + ).get_or_create("websocket:recovery-smoke") + if any( + message.get("content") == "restored without another model request" + for message in restored.messages + ): + break + time.sleep(0.1) + else: + logs = second_log.read_text(encoding="utf-8", errors="replace") + raise AssertionError(f"answer was not recovered after restart\n{logs}") + + assert restored is not None + assert PENDING_USER_TURN_KEY not in restored.metadata + assert RUNTIME_CHECKPOINT_KEY not in restored.metadata + assert restored.metadata["webui_recovery"]["reason"] == "answer_restored" + + async def assert_attach_state() -> None: + ws_url = f'{bootstrap["ws_url"]}?token={bootstrap["token"]}&client_id=recovery-smoke' + async with websockets.connect(ws_url) as ws: + await _recv_until(ws, "ready") + await ws.send(json.dumps({"type": "attach", "chat_id": "recovery-smoke"})) + attached = await _recv_until(ws, "attached") + assert attached["recovery_state"]["status"] == "recovered" + assert attached["recovery_state"]["reason"] == "answer_restored" + + asyncio.run(assert_attach_state()) + finally: + _stop_gateway(second) diff --git a/tests/webui/test_session_list_index.py b/tests/webui/test_session_list_index.py index 77819596e..4bd15427a 100644 --- a/tests/webui/test_session_list_index.py +++ b/tests/webui/test_session_list_index.py @@ -18,6 +18,7 @@ from nanobot.session.automation_turns import AUTOMATION_HISTORY_META from nanobot.session.history_visibility import HIDDEN_HISTORY_META from nanobot.session.manager import SessionManager from nanobot.session.model_selection import SESSION_MODEL_PRESET_METADATA_KEY +from nanobot.session.recovery import RECOVERY_METADATA_KEY @pytest.fixture(autouse=True) @@ -66,6 +67,30 @@ def test_webui_session_list_refreshes_after_model_preset_rename(tmp_path: Path) assert list_webui_sessions(manager)[0]["model_preset"] == "Codex" +def test_webui_session_list_surfaces_pending_recovery_state(tmp_path: Path) -> None: + manager = SessionManager(tmp_path) + session = manager.get_or_create("websocket:needs-attention") + session.add_message("user", "the interrupted task") + session.metadata[RECOVERY_METADATA_KEY] = { + "status": "awaiting_user", + "recovery_id": "recovery-123", + "reason": "uncertain_tool_state", + "attempts": 1, + # Private checkpoint details must never leak into the sidebar index. + "checkpoint": {"tool_args": "secret"}, + } + manager.save(session) + + row = list_webui_sessions(manager)[0] + + assert row["recovery_state"] == { + "status": "awaiting_user", + "recovery_id": "recovery-123", + "reason": "uncertain_tool_state", + "attempts": 1, + } + + def test_webui_session_index_uses_unique_temp_file(tmp_path: Path) -> None: manager = SessionManager(tmp_path) session = manager.get_or_create("websocket:unique-index-temp") diff --git a/tui/src/app.test.ts b/tui/src/app.test.ts index bfc89c488..375284748 100644 --- a/tui/src/app.test.ts +++ b/tui/src/app.test.ts @@ -7,7 +7,12 @@ import { } from "@opentui/core/testing" import { NanobotTui, sessionExitMessage, type AppOptions } from "./app" -import type { MessageOptions, SlashCommand, WorkspaceScopePayload } from "./protocol" +import type { + MessageOptions, + RecoveryState, + SlashCommand, + WorkspaceScopePayload, +} from "./protocol" import type { HostAgentState, HostMetadata, TuiHost } from "./host" const options: AppOptions = { @@ -91,6 +96,13 @@ function client( setWorkspaceScope(scope: WorkspaceScopePayload) { scopes.push(scope) }, + updateRecovery( + _action: "continue" | "dismiss", + _chatId: string, + recoveryId: string, + ): Promise { + return Promise.resolve({ status: "recovered" as const, recovery_id: recoveryId }) + }, } } @@ -903,6 +915,112 @@ describe("NanobotTui layout", () => { } }) + test("offers clickable recovery actions without letting a late response revive stale state", async () => { + setup = await createRenderer({ width: 96, height: 24, screenMode: "alternate-screen" }) + const calls: Array<{ action: string; chatId: string; recoveryId: string }> = [] + let deferredResolve: ((state: RecoveryState) => void) | undefined + const recoveryClient = client() + recoveryClient.updateRecovery = (action, chatId, recoveryId) => { + calls.push({ action, chatId, recoveryId }) + if (recoveryId === "recovery-1") { + return Promise.resolve({ status: "resuming", recovery_id: recoveryId }) + } + if (action === "dismiss") { + return Promise.resolve({ status: "recovered", recovery_id: recoveryId }) + } + return new Promise((resolve) => { deferredResolve = resolve }) + } + const app = NanobotTui.mount( + setup.renderer, + options, + recoveryClient, + new MockTreeSitterClient({ autoResolveTimeout: 0 }), + ) + app.accept({ + event: "attached", + chat_id: "chat", + recovery_state: { + status: "awaiting_user", + recovery_id: "recovery-1", + reason: "tool execution interrupted", + }, + }) + const ui = app as unknown as { + activeTurn: boolean + composer: TextareaRenderable + recoveryNotice: { + visible: boolean + dismiss: TextRenderable + resume: TextRenderable + } + status: TextRenderable + } + + await waitUntil(() => (app as unknown as { ready: boolean }).ready) + await setup.renderOnce() + expect(setup.captureCharFrame()).toContain("Task interrupted") + expect(ui.activeTurn).toBe(false) + expect(ui.composer.focused).toBe(true) + + await setup.mockMouse.click(ui.recoveryNotice.resume.x + 1, ui.recoveryNotice.resume.y) + await waitUntil(() => calls.length === 1 && ui.activeTurn) + expect(calls[0]).toEqual({ + action: "continue", + chatId: "chat", + recoveryId: "recovery-1", + }) + expect(ui.recoveryNotice.visible).toBe(false) + expect(ui.status.plainText).toContain("Continuing") + + app.accept({ + event: "recovery_state", + chat_id: "chat", + status: "awaiting_user", + recovery_id: "recovery-2", + }) + await setup.renderOnce() + await setup.mockMouse.click(ui.recoveryNotice.resume.x + 1, ui.recoveryNotice.resume.y) + await waitUntil(() => calls.length === 2) + app.accept({ + event: "recovery_state", + chat_id: "chat", + status: "recovered", + recovery_id: "recovery-2", + }) + deferredResolve?.({ status: "resuming", recovery_id: "recovery-2" }) + await Bun.sleep(1) + + expect(ui.recoveryNotice.visible).toBe(false) + expect(ui.activeTurn).toBe(false) + expect(ui.composer.focused).toBe(true) + + app.accept({ + event: "recovery_state", + chat_id: "chat", + status: "awaiting_user", + recovery_id: "recovery-3", + }) + await setup.renderOnce() + await setup.mockMouse.click(ui.recoveryNotice.dismiss.x + 1, ui.recoveryNotice.dismiss.y) + await waitUntil(() => calls.length === 3 && !ui.recoveryNotice.visible) + expect(calls[2]).toEqual({ + action: "dismiss", + chatId: "chat", + recoveryId: "recovery-3", + }) + + app.accept({ + event: "recovery_state", + chat_id: "chat", + status: "awaiting_user", + recovery_id: "recovery-4", + can_continue: false, + }) + await setup.renderOnce() + expect(ui.recoveryNotice.resume.visible).toBe(false) + expect(ui.recoveryNotice.dismiss.visible).toBe(true) + }) + test("preserves gateway slash lifecycle while local navigation stays in the same menu", async () => { setup = await createRenderer({ width: 80, height: 24, screenMode: "alternate-screen" }) const sent: string[] = [] diff --git a/tui/src/app.ts b/tui/src/app.ts index 4483d8516..60a2c6363 100644 --- a/tui/src/app.ts +++ b/tui/src/app.ts @@ -34,6 +34,7 @@ import { type InboundEvent, type MentionCandidate, type MessageOptions, + type RecoveryState, type SlashCommand, type SessionSummary, type TokenUsage, @@ -70,6 +71,7 @@ import { } from "./mention-menu" import { PromptQueue, type QueuedPrompt } from "./prompt-queue" import { QueuePreview, type QueuePreviewTheme } from "./queue-preview" +import { RecoveryNotice, type RecoveryNoticeTheme } from "./recovery-notice" import { RuntimeControls } from "./runtime-controls" import { contextualFooterHints, @@ -107,6 +109,11 @@ interface ChatClient { newChat(scope?: WorkspaceScopePayload): void forkChat?(sourceChatId: string, beforeUserIndex: number, title?: string): void setWorkspaceScope(scope: WorkspaceScopePayload): void + updateRecovery( + action: "continue" | "dismiss", + chatId: string, + recoveryId: string, + ): Promise } interface Palette { @@ -295,6 +302,15 @@ function queuePreviewTheme(palette: Palette): QueuePreviewTheme { } } +function recoveryNoticeTheme(palette: Palette): RecoveryNoticeTheme { + return { + text: palette.text, + muted: palette.muted, + accent: palette.accent, + error: palette.error, + } +} + function footerHintTheme(palette: Palette): FooterHintTheme { return { accent: palette.accent, @@ -380,6 +396,7 @@ export class NanobotTui { private readonly contextPanel: ContextPanel private readonly diffViewer: DiffViewer private readonly queuePreview: QueuePreview + private readonly recoveryNotice: RecoveryNotice private readonly client: ChatClient private readonly shell: BoxRenderable private readonly title: BoxRenderable @@ -444,6 +461,8 @@ export class NanobotTui { private currentTask = "" private currentAction = "" private hostBlocked = false + private recoveryState: RecoveryState | null = null + private recoveryPending = false private hostWorkspace: string private hostBranch: string private readonly apiReauthenticator: ApiReauthenticator | undefined @@ -495,6 +514,14 @@ export class NanobotTui { treeSitterClient, ) this.queuePreview = new QueuePreview(renderer, queuePreviewTheme(this.palette)) + this.recoveryNotice = new RecoveryNotice( + renderer, + recoveryNoticeTheme(this.palette), + { + onContinue: () => void this.updateRecovery("continue"), + onDismiss: () => void this.updateRecovery("dismiss"), + }, + ) this.client = client || new NanobotClient({ ...(options.bootstrapUrl ? { @@ -723,6 +750,7 @@ export class NanobotTui { this.shell.add(this.runtimeControls.menuRoot) if (!host.hosted) this.shell.add(this.title) this.shell.add(this.queuePreview.root) + this.shell.add(this.recoveryNotice.root) this.shell.add(this.composerFrame) this.shell.add(statusRow) this.shell.add(this.diffViewer.root) @@ -831,6 +859,12 @@ export class NanobotTui { this.quit() return } + if (["/continue", "/dismiss"].includes(visibleContent.toLowerCase())) { + this.clearComposer() + this.commandMenu.hide() + void this.updateRecovery(visibleContent.toLowerCase() === "/continue" ? "continue" : "dismiss") + return + } const completion = this.commandMenu.completion(visibleContent) if (completion) { this.setComposer(completion) @@ -946,7 +980,9 @@ export class NanobotTui { } const hydrationId = ++this.hydrationId void this.prepareChat(event.chat_id, restoring, hydrationId).then(() => { - if (hydrationId === this.hydrationId) this.flushPendingEvents() + if (hydrationId !== this.hydrationId) return + this.applyRecoveryState(event.recovery_state ?? null) + this.flushPendingEvents() }) return } @@ -1083,6 +1119,9 @@ export class NanobotTui { this.applyHostGoalState(event.goal_state) if (!this.activeTurn) this.reportHostResting() return + case "recovery_state": + this.applyRecoveryState(event) + return case "turn_model_updated": if (typeof event.context_window_tokens === "number") { this.contextWindowTokens = event.context_window_tokens @@ -1191,6 +1230,85 @@ export class NanobotTui { for (const event of events || []) this.accept(event) } + private clearRecoveryState(): void { + this.recoveryState = null + this.recoveryPending = false + this.recoveryNotice.hide() + } + + private applyRecoveryState(state: RecoveryState | null): void { + if (!state) { + this.clearRecoveryState() + return + } + this.recoveryState = state + this.recoveryPending = false + if (state.status === "resuming") { + this.recoveryNotice.hide() + this.hostBlocked = false + this.activeLabel = "Continuing" + this.setCurrentAction("Continuing interrupted task") + this.setActive(true) + this.reportHostWorking() + return + } + if (state.status === "awaiting_user" || state.status === "failed") { + this.activeTurnId = null + this.setActive(false) + this.hostBlocked = true + this.recoveryNotice.show(state) + const detail = state.reason || (state.status === "failed" + ? "Recovery failed" + : "Task interrupted") + this.setCurrentAction(detail) + this.status.content = "Waiting for recovery decision" + this.host.reportState("blocked", detail) + this.composer.focus() + return + } + this.clearRecoveryState() + this.activeTurnId = null + this.hostBlocked = false + this.setActive(false) + if (this.ready) this.status.content = this.readyStatus() + this.reportHostResting() + } + + private async updateRecovery(action: "continue" | "dismiss"): Promise { + const state = this.recoveryState + if ( + !state + || (state.status !== "awaiting_user" && state.status !== "failed") + || (action === "continue" && state.can_continue === false) + ) { + this.status.content = "No interrupted task" + this.composer.focus() + return + } + if (this.recoveryPending) return + this.recoveryPending = true + this.recoveryNotice.setBusy(true) + this.status.content = action === "continue" ? "Continuing…" : "Dismissing…" + try { + const next = await this.client.updateRecovery( + action, + this.client.activeChatId, + state.recovery_id, + ) + if (this.recoveryState?.recovery_id === state.recovery_id) { + this.applyRecoveryState(next) + } + } catch (error) { + if (this.recoveryState?.recovery_id !== state.recovery_id) return + this.recoveryPending = false + this.recoveryNotice.setBusy(false) + this.status.content = error instanceof Error ? error.message : String(error) + this.host.reportState("blocked", state.reason || "Task interrupted") + } finally { + this.composer.focus() + } + } + private updateGatewayApiConnection(apiUrl: string, apiToken: string): void { this.options.apiUrl = apiUrl this.options.apiToken = apiToken @@ -1576,6 +1694,7 @@ export class NanobotTui { this.contextPanel.setTheme(contextPanelTheme(this.palette)) this.diffViewer.setTheme(diffViewerTheme(this.palette, this.backgroundKnown)) this.queuePreview.setTheme(queuePreviewTheme(this.palette)) + this.recoveryNotice.setTheme(recoveryNoticeTheme(this.palette)) this.updateComposerAppearance() this.composer.textColor = this.palette.text this.composer.focusedTextColor = this.palette.text @@ -2001,6 +2120,7 @@ export class NanobotTui { this.sessionTitle = sessionLabel(current) this.applySessionModel(current) this.applySessionScope(current) + this.applyRecoveryState(current.recoveryState ?? null) this.updateTitle() } const limit = this.renderer.height >= 20 ? 8 : 4 @@ -2027,6 +2147,7 @@ export class NanobotTui { this.sessionTitle = sessionLabel(session) this.applySessionModel(session) this.applySessionScope(session) + this.applyRecoveryState(session.recoveryState ?? null) this.updateTitle() this.closeSessions() this.status.content = this.readyStatus() @@ -2039,6 +2160,7 @@ export class NanobotTui { this.closeSessions() try { this.ready = false + this.clearRecoveryState() this.clearPromptQueue() this.sessionMetadataId += 1 this.clearHostContext() @@ -2073,6 +2195,7 @@ export class NanobotTui { this.clearComposer() try { this.ready = false + this.clearRecoveryState() this.clearPromptQueue() this.sessionMetadataId += 1 this.clearHostContext() diff --git a/tui/src/protocol.test.ts b/tui/src/protocol.test.ts index 36686b41b..4ff1cbafc 100644 --- a/tui/src/protocol.test.ts +++ b/tui/src/protocol.test.ts @@ -595,6 +595,62 @@ describe("gateway protocol", () => { } }) + test("sends stale-safe recovery mutations and validates their response", async () => { + const original = globalThis.WebSocket + let socket: FakeSocket | undefined + Object.defineProperty(globalThis, "WebSocket", { + configurable: true, + value: class extends FakeSocket { + constructor() { + super() + socket = this + } + }, + }) + + try { + const statuses: string[] = [] + const client = new NanobotClient({ + url: "ws://nanobot.test/ws", + onEvent: () => undefined, + onStatus: (status, detail) => statuses.push(`${status}:${detail || ""}`), + }) + client.connect() + if (!socket) throw new Error("socket was not created") + + const result = client.updateRecovery("continue", "chat", "recovery-1") + const request = JSON.parse(socket.sent.at(-1) || "{}") as { + request_id: string + action: string + payload: Record + } + expect(request).toMatchObject({ + type: "webui_request", + action: "recovery.continue", + payload: { chat_id: "chat", recovery_id: "recovery-1" }, + }) + socket.emit("message", { data: JSON.stringify({ + event: "webui_response", + request_id: request.request_id, + ok: true, + result: { status: "resuming", recovery_id: "recovery-1", attempts: 1 }, + }) }) + expect(await result).toEqual({ + status: "resuming", + recovery_id: "recovery-1", + attempts: 1, + }) + expect(statuses).not.toContain("error:gateway sent an invalid event") + + const interrupted = client.updateRecovery("dismiss", "chat", "recovery-2") + socket.emit("close") + await expect(interrupted).rejects.toThrow("gateway connection closed") + client.close() + } finally { + Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original }) + } + }) + test("reports when the bounded history snapshot omits earlier turns", async () => { const original = globalThis.fetch let requested = "" diff --git a/tui/src/protocol.ts b/tui/src/protocol.ts index abd4cb2f6..3d1237b71 100644 --- a/tui/src/protocol.ts +++ b/tui/src/protocol.ts @@ -54,6 +54,16 @@ export interface RuntimeControls { canUseFullAccess: boolean } +export type RecoveryStatus = "resuming" | "awaiting_user" | "recovered" | "failed" + +export interface RecoveryState { + status: RecoveryStatus + recovery_id: string + reason?: string + attempts?: number + can_continue?: boolean +} + export type InboundEvent = | { event: "ready"; chat_id: string; client_id: string } | { @@ -61,6 +71,7 @@ export type InboundEvent = chat_id: string model_preset?: string | null usage?: TokenUsage + recovery_state?: RecoveryState } | { event: "message_accepted" @@ -118,6 +129,7 @@ export type InboundEvent = turn_id?: string } | { event: "goal_state"; chat_id: string; goal_state: Record } + | ({ event: "recovery_state"; chat_id: string } & RecoveryState) | { event: "session_updated" chat_id: string @@ -139,6 +151,12 @@ type OutboundEvent = | { type: "fork_chat"; source_chat_id: string; before_user_index: number; title?: string } | { type: "attach"; chat_id: string } | { type: "set_workspace_scope"; chat_id: string; workspace_scope: WorkspaceScopePayload } + | { + type: "webui_request" + request_id: string + action: string + payload: Record + } | { type: "message" chat_id: string @@ -271,6 +289,7 @@ export interface SessionSummary { updatedAt: string | null runStartedAt: number | null modelPreset: string | null + recoveryState?: RecoveryState | null workspaceScope?: WorkspaceScopePayload | null pinned: boolean archived: boolean @@ -297,6 +316,7 @@ const CHAT_EVENTS = new Set([ "turn_end", "goal_status", "goal_state", + "recovery_state", "session_updated", "turn_model_updated", "error", @@ -377,6 +397,34 @@ function isWorkspaceScope(value: unknown): value is WorkspaceScopePayload { && optional(value.restrict_to_workspace, "boolean") } +function isRecoveryState(value: unknown): value is RecoveryState { + return isRecord(value) + && ["resuming", "awaiting_user", "recovered", "failed"].includes(String(value.status)) + && typeof value.recovery_id === "string" + && optional(value.reason, "string") + && optional(value.attempts, "number") + && optional(value.can_continue, "boolean") +} + +interface WebUIResponseEvent { + event: "webui_response" + request_id: string + ok: boolean + result?: unknown + error?: { status: number; message: string } +} + +function decodeWebUIResponse(value: unknown): WebUIResponseEvent | null | undefined { + if (!isRecord(value) || value.event !== "webui_response") return undefined + if (typeof value.request_id !== "string" || typeof value.ok !== "boolean") return null + if (value.ok) return value as unknown as WebUIResponseEvent + return isRecord(value.error) + && typeof value.error.status === "number" + && typeof value.error.message === "string" + ? value as unknown as WebUIResponseEvent + : null +} + function decodeInboundEvent(value: unknown): InboundEvent | null | undefined { if (!isRecord(value)) return null const record = value @@ -407,7 +455,8 @@ function decodeInboundEvent(value: unknown): InboundEvent | null | undefined { && ((record.model_preset !== undefined && record.model_preset !== null && typeof record.model_preset !== "string") - || (record.usage !== undefined && !isTokenUsage(record.usage))) + || (record.usage !== undefined && !isTokenUsage(record.usage)) + || (record.recovery_state !== undefined && !isRecoveryState(record.recovery_state))) ) return null if ( ["user_message", "message", "delta", "reasoning_delta"].includes(name) @@ -452,6 +501,7 @@ function decodeInboundEvent(value: unknown): InboundEvent | null | undefined { ) return null if (name === "goal_status" && record.status !== "running" && record.status !== "idle") return null if (name === "goal_state" && !isRecord(record.goal_state)) return null + if (name === "recovery_state" && !isRecoveryState(record)) return null if ( name === "session_updated" && (!optional(record.scope, "string") @@ -700,6 +750,9 @@ export async function fetchSessions( modelPreset: typeof value.model_preset === "string" && value.model_preset.trim() ? value.model_preset.trim() : null, + ...(isRecoveryState(value.recovery_state) + ? { recoveryState: value.recovery_state } + : {}), ...(isWorkspaceScope(value.workspace_scope) ? { workspaceScope: value.workspace_scope } : {}), pinned: pinned.has(value.key), archived: archived.has(value.key), @@ -855,6 +908,11 @@ export class NanobotClient { private closedByClient = false private opening = false private connectedOnce = false + private readonly pendingMutations = new Map void + reject: (error: Error) => void + timer: ReturnType + }>() constructor(private readonly options: ClientOptions) {} @@ -916,6 +974,7 @@ export class NanobotClient { socket.addEventListener("close", () => { if (this.socket !== socket) return this.socket = null + this.rejectPendingMutations("gateway connection closed") if (this.closedByClient) { this.options.onStatus("closed") return @@ -931,6 +990,7 @@ export class NanobotClient { const socket = this.socket this.socket = null socket?.close() + this.rejectPendingMutations("gateway connection closed") } send(content: string, options: MessageOptions = {}): string { @@ -975,6 +1035,63 @@ export class NanobotClient { this.write({ type: "set_workspace_scope", chat_id: this.chatId, workspace_scope: scope }) } + updateRecovery( + action: "continue" | "dismiss", + chatId: string, + recoveryId: string, + ): Promise { + return this.requestMutation(`recovery.${action}`, { + chat_id: chatId, + recovery_id: recoveryId, + }).then((result) => { + if (!isRecoveryState(result)) throw new Error("gateway returned an invalid recovery state") + return result + }) + } + + private requestMutation( + action: string, + payload: Record = {}, + timeoutMs = 20_000, + ): Promise { + if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { + return Promise.reject(new Error("gateway connection is not open")) + } + const requestId = crypto.randomUUID() + const frame = JSON.stringify({ + type: "webui_request", + request_id: requestId, + action, + payload, + } satisfies OutboundEvent) + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pendingMutations.delete(requestId) + reject(new Error(`gateway request timed out after ${timeoutMs}ms`)) + }, timeoutMs) + this.pendingMutations.set(requestId, { + resolve: (value) => resolve(value as T), + reject, + timer, + }) + try { + this.socket?.send(frame) + } catch { + clearTimeout(timer) + this.pendingMutations.delete(requestId) + reject(new Error("could not send gateway request")) + } + }) + } + + private rejectPendingMutations(message: string): void { + for (const pending of this.pendingMutations.values()) { + clearTimeout(pending.timer) + pending.reject(new Error(message)) + } + this.pendingMutations.clear() + } + private handleMessage(raw: string): void { let value: unknown try { @@ -983,6 +1100,20 @@ export class NanobotClient { this.options.onStatus("error", "gateway sent invalid JSON") return } + const response = decodeWebUIResponse(value) + if (response === null) { + this.options.onStatus("error", "gateway sent an invalid event") + return + } + if (response) { + const pending = this.pendingMutations.get(response.request_id) + if (!pending) return + clearTimeout(pending.timer) + this.pendingMutations.delete(response.request_id) + if (response.ok) pending.resolve(response.result) + else pending.reject(new Error(response.error?.message || "gateway request failed")) + return + } const event = decodeInboundEvent(value) if (event === undefined) return if (event === null) { diff --git a/tui/src/recovery-notice.ts b/tui/src/recovery-notice.ts new file mode 100644 index 000000000..8403526a3 --- /dev/null +++ b/tui/src/recovery-notice.ts @@ -0,0 +1,155 @@ +import { + BoxRenderable, + RGBA, + StyledText, + TextAttributes, + TextRenderable, + type CliRenderer, + type TextChunk, +} from "@opentui/core" + +import type { RecoveryState } from "./protocol" + +export interface RecoveryNoticeTheme { + text: string + muted: string + accent: string + error: string +} + +interface RecoveryNoticeOptions { + onContinue: () => void + onDismiss: () => void +} + +/** A quiet action surface for a gateway-owned interrupted turn. */ +export class RecoveryNotice { + readonly root: BoxRenderable + private readonly message: TextRenderable + private readonly dismiss: TextRenderable + private readonly resume: TextRenderable + private state: RecoveryState | null = null + private busy = false + + constructor( + renderer: CliRenderer, + private theme: RecoveryNoticeTheme, + options: RecoveryNoticeOptions, + ) { + this.root = new BoxRenderable(renderer, { + id: "nanobot-tui-recovery-notice", + width: "100%", + height: 1, + flexShrink: 0, + flexDirection: "row", + alignItems: "center", + gap: 2, + paddingLeft: 1, + paddingRight: 1, + visible: false, + backgroundColor: RGBA.defaultBackground(), + }) + this.message = new TextRenderable(renderer, { + id: "nanobot-tui-recovery-message", + width: "auto", + minWidth: 0, + flexGrow: 1, + height: 1, + truncate: true, + selectable: false, + }) + this.dismiss = this.action(renderer, "dismiss", "Dismiss", options.onDismiss) + this.resume = this.action(renderer, "continue", "Continue", options.onContinue, true) + this.root.add(this.message) + this.root.add(this.dismiss) + this.root.add(this.resume) + } + + get visible(): boolean { + return this.root.visible + } + + show(state: RecoveryState): void { + this.state = state + this.busy = false + this.root.visible = true + this.render() + } + + hide(): void { + this.state = null + this.busy = false + this.root.visible = false + } + + setBusy(busy: boolean): void { + this.busy = busy + if (this.visible) this.render() + } + + setTheme(theme: RecoveryNoticeTheme): void { + this.theme = theme + if (this.visible) this.render() + } + + private action( + renderer: CliRenderer, + id: string, + label: string, + callback: () => void, + primary = false, + ): TextRenderable { + return new TextRenderable(renderer, { + id: `nanobot-tui-recovery-${id}`, + content: label, + width: label.length, + height: 1, + flexShrink: 0, + selectable: false, + onMouseOver: () => { + if (this.busy) return + const target = primary ? this.resume : this.dismiss + target.attributes = TextAttributes.BOLD | TextAttributes.UNDERLINE + }, + onMouseOut: () => this.render(), + onMouseDown: (event) => { + if (event.button !== 0 || this.busy) return + event.preventDefault() + event.stopPropagation() + renderer.clearSelection() + callback() + }, + }) + } + + private render(): void { + if (!this.state) return + const failed = this.state.status === "failed" + const contextUnavailable = this.state.can_continue === false + const title = failed ? "Recovery failed" : "Task interrupted" + const detail = failed + ? "Review the task before continuing" + : contextUnavailable + ? "Saved context unavailable" + : "Tools will not replay automatically" + this.message.content = new StyledText([ + chunk("△ ", failed ? this.theme.error : this.theme.accent), + chunk(title, this.theme.text, true), + chunk(` · ${detail}`, this.theme.muted), + ]) + this.dismiss.fg = RGBA.fromHex(this.busy ? this.theme.muted : this.theme.text) + this.resume.visible = !contextUnavailable + this.resume.fg = RGBA.fromHex(this.busy ? this.theme.muted : this.theme.accent) + this.dismiss.attributes = 0 + this.resume.attributes = TextAttributes.BOLD + } +} + +function chunk(text: string, color: string, bold = false): TextChunk { + return { + __isChunk: true, + text, + fg: RGBA.fromHex(color), + attributes: bold ? TextAttributes.BOLD : 0, + } +} diff --git a/tui/src/session-menu.test.ts b/tui/src/session-menu.test.ts index 832d1ff58..afdc27aff 100644 --- a/tui/src/session-menu.test.ts +++ b/tui/src/session-menu.test.ts @@ -24,6 +24,11 @@ const sessions: SessionSummary[] = [ updatedAt: "2026-08-12T10:00:00Z", runStartedAt: null, modelPreset: null, + recoveryState: { + status: "awaiting_user", + recovery_id: "recovery-two", + reason: "tool execution interrupted", + }, pinned: false, archived: false, }, @@ -53,7 +58,7 @@ describe("SessionMenu", () => { menu.update("release stable", 6) await setup.renderOnce() - expect(setup.captureCharFrame()).toContain("Release checklist") + expect(setup.captureCharFrame()).toContain("△ Release checklist") expect(menu.choose()?.chatId).toBe("two") }) diff --git a/tui/src/session-menu.ts b/tui/src/session-menu.ts index 9dde91e3a..ee456cc7d 100644 --- a/tui/src/session-menu.ts +++ b/tui/src/session-menu.ts @@ -42,6 +42,8 @@ export class SessionMenu { session.chatId, session.workspaceScope?.project_name || "", session.workspaceScope?.project_path || "", + session.recoveryState?.status || "", + session.recoveryState?.reason || "", ].join(" "), render: (session) => { const age = updatedLabel(session.updatedAt) @@ -54,7 +56,11 @@ export class SessionMenu { ] .filter(Boolean) .join(" · ") - const marker = session.active ? "● " : session.pinned ? "◆ " : session.archived ? "◇ " : "" + const interrupted = session.recoveryState?.status === "awaiting_user" + || session.recoveryState?.status === "failed" + const marker = interrupted + ? "△ " + : session.active ? "● " : session.pinned ? "◆ " : session.archived ? "◇ " : "" return `${marker}${sessionLabel(session)}${detail ? ` ${detail}` : ""}` }, emptyText: "No matching sessions", diff --git a/webui/src/App.tsx b/webui/src/App.tsx index bae5e526c..d5f31a6ca 100644 --- a/webui/src/App.tsx +++ b/webui/src/App.tsx @@ -1274,6 +1274,15 @@ function Shell({ }, [activeKey, activeTabKey, activeTabState]); const runningChatIdList = useMemo(() => Array.from(runningChatIds), [runningChatIds]); const updatedChatIdList = useMemo(() => Array.from(updatedChatIds), [updatedChatIds]); + const recoveryChatIdList = useMemo( + () => sessions + .filter((session) => ( + session.recoveryState?.status === "awaiting_user" + || session.recoveryState?.status === "failed" + )) + .map((session) => session.chatId), + [sessions], + ); const activeChatId = activePaneSession?.chatId ?? null; useEffect(() => { activeChatIdRef.current = activeChatId; @@ -2552,6 +2561,7 @@ function Shell({ collapsedGroups: sidebarState.collapsed_groups, runningChatIds: runningChatIdList, updatedChatIds: updatedChatIdList, + recoveryChatIds: recoveryChatIdList, viewState: { ...sidebarState.view, sort: automaticSidebarSort }, showArchived: sidebarState.view.show_archived, archivedCount: sidebarArchivedTabKeys.length, diff --git a/webui/src/components/ChatList.tsx b/webui/src/components/ChatList.tsx index 9cfb92654..2ace0e617 100644 --- a/webui/src/components/ChatList.tsx +++ b/webui/src/components/ChatList.tsx @@ -11,6 +11,7 @@ import type { CSSProperties, MouseEvent as ReactMouseEvent, ReactElement } from import { Archive, ArchiveRestore, + AlertTriangle, ChevronDown, Folder, FolderTree, @@ -260,6 +261,7 @@ interface ChatListProps { collapsedGroups?: Record; runningChatIds?: string[]; updatedChatIds?: string[]; + recoveryChatIds?: string[]; density?: SidebarDensity; showPreviews?: boolean; showTimestamps?: boolean; @@ -302,6 +304,7 @@ export const ChatList = memo(function ChatList({ collapsedGroups = {}, runningChatIds = [], updatedChatIds = [], + recoveryChatIds = [], density = "comfortable", showPreviews = false, showTimestamps = false, @@ -558,6 +561,7 @@ export const ChatList = memo(function ChatList({ const running = new Set(runningChatIds); const updated = new Set(updatedChatIds); + const recovery = new Set(recoveryChatIds); const compact = density === "compact"; const firstProjectGroupIndex = limitedGroups.findIndex((group) => group.kind === "project"); const selectableDeleteKeys = Array.from(new Set(limitedGroups.flatMap((group) => ( @@ -881,6 +885,7 @@ export const ChatList = memo(function ChatList({ compact={compact} running={running} updated={updated} + recovery={recovery} onSelectPane={onSelectPane} onRequestDelete={onRequestDelete} onRequestRename={onRequestRename} @@ -915,9 +920,11 @@ export const ChatList = memo(function ChatList({ : ""; const activityState = running.has(s.chatId) ? "running" - : updated.has(s.chatId) && !topicActive - ? "updated" - : null; + : recovery.has(s.chatId) + ? "recovery" + : updated.has(s.chatId) && !topicActive + ? "updated" + : null; const hasPaneMoveTarget = Boolean(onAttachPane) && paneGroupTargets.some((target) => ( target.key !== paneGroup?.tabKey && !target.atCapacity @@ -1330,6 +1337,7 @@ function ActivePaneRows({ compact, running, updated, + recovery, onSelectPane, onRequestDelete, onRequestRename, @@ -1354,6 +1362,7 @@ function ActivePaneRows({ compact: boolean; running: ReadonlySet; updated: ReadonlySet; + recovery: ReadonlySet; onSelectPane?: (tabKey: string, paneKey: string) => void; onRequestDelete: (key: string, label: string) => void; onRequestRename: (key: string, label: string) => void; @@ -1390,9 +1399,11 @@ function ActivePaneRows({ const active = tabActive && pane.key === group.activePaneKey; const activityState = running.has(pane.chatId) ? "running" - : updated.has(pane.chatId) && !active - ? "updated" - : null; + : recovery.has(pane.chatId) + ? "recovery" + : updated.has(pane.chatId) && !active + ? "updated" + : null; const paneActionsLabel = t("workbench.paneActions", { title: pane.title }); const selected = selectedDeleteKeys.has(pane.key); const isPinned = pinned.has(pane.key); @@ -1852,10 +1863,27 @@ function ChatsFoldFooter({ function SessionActivityIndicator({ state, }: { - state: "running" | "updated" | null; + state: "running" | "updated" | "recovery" | null; }) { const { t } = useTranslation(); + if (state === "recovery") { + const label = t("chat.activity.recovery", { + defaultValue: "This conversation needs your attention", + }); + return ( + + + + + + ); + } + if (state === "running") { const label = t("chat.activity.running"); return ( diff --git a/webui/src/components/Sidebar.tsx b/webui/src/components/Sidebar.tsx index 28e5f5cd2..1a066ee9a 100644 --- a/webui/src/components/Sidebar.tsx +++ b/webui/src/components/Sidebar.tsx @@ -82,6 +82,7 @@ interface SidebarProps { collapsedGroups?: Record; runningChatIds?: string[]; updatedChatIds?: string[]; + recoveryChatIds?: string[]; viewState?: SidebarViewState; showArchived?: boolean; archivedCount?: number; @@ -270,6 +271,7 @@ export function Sidebar(props: SidebarProps) { collapsedGroups={props.collapsedGroups} runningChatIds={props.runningChatIds} updatedChatIds={props.updatedChatIds} + recoveryChatIds={props.recoveryChatIds} density={props.viewState?.density} showPreviews={props.viewState?.show_previews} showTimestamps={props.viewState?.show_timestamps} diff --git a/webui/src/components/settings/overview/OverviewSettings.tsx b/webui/src/components/settings/overview/OverviewSettings.tsx index 33bc62ce8..e804b1be9 100644 --- a/webui/src/components/settings/overview/OverviewSettings.tsx +++ b/webui/src/components/settings/overview/OverviewSettings.tsx @@ -421,6 +421,37 @@ export function AppearanceSettings({ label={localPrefs.brandLogos ? tx("settings.values.on", "On") : tx("settings.values.off", "Off")} /> + + { + if (!enabled) { + onChangeLocalPrefs((prev) => ({ ...prev, browserNotifications: false })); + return; + } + if (typeof Notification === "undefined") return; + if (Notification.permission === "granted") { + onChangeLocalPrefs((prev) => ({ ...prev, browserNotifications: true })); + return; + } + void Notification.requestPermission().then((permission) => { + if (permission === "granted") { + onChangeLocalPrefs((prev) => ({ ...prev, browserNotifications: true })); + } + }); + }} + ariaLabel={tx("settings.rows.browserNotifications", "Task notifications")} + label={localPrefs.browserNotifications + ? tx("settings.values.on", "On") + : tx("settings.values.off", "Off")} + /> + diff --git a/webui/src/components/thread/RecoveryNotice.tsx b/webui/src/components/thread/RecoveryNotice.tsx new file mode 100644 index 000000000..1b8e8660c --- /dev/null +++ b/webui/src/components/thread/RecoveryNotice.tsx @@ -0,0 +1,108 @@ +import { useEffect, useState } from "react"; +import { AlertTriangle, LoaderCircle, RotateCcw, X } from "lucide-react"; +import { useTranslation } from "react-i18next"; + +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; +import type { RecoveryState } from "@/lib/types"; + +interface RecoveryNoticeProps { + state: RecoveryState; + onContinue: () => Promise; + onDismiss: () => Promise; +} + +export function RecoveryNotice({ state, onContinue, onDismiss }: RecoveryNoticeProps) { + const { t } = useTranslation(); + const [pending, setPending] = useState<"continue" | "dismiss" | null>(null); + const [error, setError] = useState(null); + const [hiddenRecoveryId, setHiddenRecoveryId] = useState(null); + useEffect(() => { + // A continuation can be interrupted again with the same recovery ID. + // Reveal the decision surface when the server returns to a waiting state. + if (state.status === "awaiting_user" || state.status === "failed") { + setHiddenRecoveryId(null); + } + }, [state.recovery_id, state.status]); + if (state.status === "recovered" || hiddenRecoveryId === state.recovery_id) return null; + + const waiting = state.status === "awaiting_user" || state.status === "failed"; + const contextUnavailable = state.can_continue === false; + const title = state.status === "failed" + ? t("recovery.failed", { defaultValue: "Task recovery failed" }) + : waiting + ? t("recovery.interrupted", { defaultValue: "Task interrupted" }) + : t("recovery.resuming", { defaultValue: "Restoring interrupted task…" }); + const detail = state.status === "failed" || contextUnavailable + ? t("recovery.failedHelp", { + defaultValue: "The saved task could not be restored safely. Review it before continuing.", + }) + : waiting + ? t("recovery.review", { defaultValue: "Review the task before continuing. Tools will not be replayed automatically." }) + : t("recovery.safeResume", { defaultValue: "Continuing from saved conversation context." }); + const run = (action: "continue" | "dismiss") => { + setPending(action); + setError(null); + // ``resuming`` is an internal transition, not another task for the user + // to monitor. Hide the notice optimistically and only bring it back if + // the explicit action is rejected. + if (action === "continue") setHiddenRecoveryId(state.recovery_id); + const operation = action === "continue" ? onContinue() : onDismiss(); + void operation.catch(() => { + if (action === "continue") setHiddenRecoveryId(null); + setError(t("recovery.actionFailed", { defaultValue: "Recovery action failed. Try again." })); + }).finally(() => setPending(null)); + }; + + return ( +
+ {waiting ? ( + + ) : ( + + )} +
+

+ {title} +

+

+ {error ?? detail} +

+
+ {waiting ? ( +
+ + {!contextUnavailable ? ( + + ) : null} +
+ ) : null} +
+ ); +} diff --git a/webui/src/components/thread/ThreadShell.tsx b/webui/src/components/thread/ThreadShell.tsx index 172271601..273b56c57 100644 --- a/webui/src/components/thread/ThreadShell.tsx +++ b/webui/src/components/thread/ThreadShell.tsx @@ -7,6 +7,7 @@ import { FilePreviewAvailabilityProvider } from "@/components/FilePreviewAvailab import { FilePreviewPanel } from "@/components/FilePreviewPanel"; import { SessionHandleLabel } from "@/components/SessionHandleLabel"; import { PromptNavigator } from "@/components/thread/PromptNavigator"; +import { RecoveryNotice } from "@/components/thread/RecoveryNotice"; import { SessionInfoPopover } from "@/components/thread/SessionInfoPopover"; import { ThreadComposer, @@ -763,6 +764,9 @@ export function ThreadShell({ isStreaming, runStartedAt, goalState, + recoveryState, + continueRecovery, + dismissRecovery, send, transcribeAudio, stop, @@ -835,8 +839,15 @@ export function ThreadShell({ [displayMessages], ); const currentGoalState = messagesReady ? goalState : undefined; - const currentRunStartedAt = messagesReady ? runStartedAt : null; - const turnActive = messagesReady && (isStreaming || currentRunStartedAt !== null); + // Decision states freeze the interrupted turn and hand the next action to + // the recovery notice. ``resuming`` remains active; ``recovered`` is only + // historical metadata and must not suppress a later normal turn. + const recoveryNeedsDecision = recoveryState?.status === "awaiting_user" + || recoveryState?.status === "failed"; + const currentRunStartedAt = messagesReady && !recoveryNeedsDecision ? runStartedAt : null; + const turnActive = messagesReady + && !recoveryNeedsDecision + && (isStreaming || currentRunStartedAt !== null); const restoredViewportTurnId = useMemo( () => turnActive ? latestActiveTurnId(displayMessages, currentRunStartedAt) : null, [currentRunStartedAt, displayMessages, turnActive], @@ -1472,6 +1483,13 @@ export function ThreadShell({ const composer = ( <> + {recoveryState ? ( + + ) : null} {streamError && !hasInlineDeliveryError(messages, streamError) ? ( Promise; + dismissRecovery: () => Promise; send: ( content: string, images?: SendAttachment[], @@ -262,6 +268,7 @@ export function useNanobotStream( dismissStreamError: () => void; } { const { client } = useClient(); + const { t } = useTranslation(); const initialRunStartedAt = chatId ? client.getRunStartedAt(chatId) : null; const [messages, setMessages] = useState(initialMessages); const [messageOwnerChatId, setMessageOwnerChatId] = useState(chatId); @@ -273,6 +280,7 @@ export function useNanobotStream( /** Unix epoch seconds when the current user turn started; cleared on ``idle``. */ const [runStartedAt, setRunStartedAt] = useState(initialRunStartedAt); const [goalState, setGoalState] = useState(undefined); + const [recoveryState, setRecoveryState] = useState(null); const [streamError, setStreamError] = useState(null); const buffer = useRef(null); const activeAssistantRef = useRef(null); @@ -288,6 +296,16 @@ export function useNanobotStream( const dismissStreamError = useCallback(() => setStreamError(null), []); + const notifyInBackground = useCallback((body: string) => { + if ( + typeof Notification === "undefined" + || Notification.permission !== "granted" + || document.visibilityState === "visible" + || !readLocalPreferences().browserNotifications + ) return; + new Notification("nanobot", { body }); + }, []); + const clearPendingStreamWork = useCallback(() => { if (streamFrameRef.current !== null) { window.cancelAnimationFrame(streamFrameRef.current); @@ -639,6 +657,7 @@ export function useNanobotStream( setStreamError(null); setRunStartedAt(restoredRunStartedAt); setGoalState(chatId ? client.getGoalState(chatId) : undefined); + setRecoveryState(null); buffer.current = null; activeAssistantRef.current = null; closedAssistantStreamIdsRef.current.clear(); @@ -846,10 +865,71 @@ export function useNanobotStream( return finalized; }); suppressStreamUntilTurnEndRef.current = false; + notifyInBackground(t("recovery.completed", { defaultValue: "Task completed" })); onTurnEnd?.(); return; } + if (ev.event === "recovery_state") { + const next: RecoveryState = { + status: ev.status, + recovery_id: ev.recovery_id, + ...(ev.reason ? { reason: ev.reason } : {}), + ...(typeof ev.attempts === "number" ? { attempts: ev.attempts } : {}), + ...(typeof ev.can_continue === "boolean" + ? { can_continue: ev.can_continue } + : {}), + }; + setRecoveryState(next); + if (ev.status === "resuming") { + setRunStartedAt((current) => current ?? Date.now() / 1000); + setIsStreaming(true); + } + if ( + ev.status === "awaiting_user" + || ev.status === "recovered" + || ev.status === "failed" + ) { + // Recovery is an explicit boundary. The interrupted turn is no + // longer running, so do not let the stale start time keep the + // activity clock (or composer stop state) alive underneath the + // recovery notice. + setRunStartedAt(null); + setIsStreaming(false); + client.finishRunLocally(chatId); + clearPendingStreamWork(); + closeActiveAssistantStream(); + clearActivitySegment(); + if (ev.status !== "recovered") { + notifyInBackground( + ev.status === "failed" + ? t("recovery.failed", { defaultValue: "Task recovery failed" }) + : t("recovery.interrupted", { defaultValue: "Task interrupted" }), + ); + } + } + return; + } + + if (ev.event === "attached") { + setRecoveryState(ev.recovery_state ?? null); + if (ev.recovery_state?.status === "resuming") { + setRunStartedAt((current) => current ?? Date.now() / 1000); + setIsStreaming(true); + } else if ( + ev.recovery_state?.status === "awaiting_user" + || ev.recovery_state?.status === "failed" + ) { + setRunStartedAt(null); + setIsStreaming(false); + client.finishRunLocally(chatId); + clearPendingStreamWork(); + closeActiveAssistantStream(); + clearActivitySegment(); + } + return; + } + if (ev.event === "message") { if ( suppressStreamUntilTurnEndRef.current && @@ -1062,8 +1142,10 @@ export function useNanobotStream( ensureActivitySegmentId, flushPendingStreamEvents, isSideChannelEvent, + notifyInBackground, onTurnEnd, schedulePendingStreamFlush, + t, ]); const send = useCallback( @@ -1173,12 +1255,32 @@ export function useNanobotStream( [client], ); + const recoveryAction = useCallback(async (action: "continue" | "dismiss") => { + if (!chatId || !recoveryState) return; + await client.requestMutation(`recovery.${action}`, { + chat_id: chatId, + recovery_id: recoveryState.recovery_id, + }); + }, [chatId, client, recoveryState]); + + const continueRecovery = useCallback( + () => recoveryAction("continue"), + [recoveryAction], + ); + const dismissRecovery = useCallback( + () => recoveryAction("dismiss"), + [recoveryAction], + ); + return { messages, messagesReady: messageOwnerChatId === chatId, isStreaming, runStartedAt, goalState, + recoveryState, + continueRecovery, + dismissRecovery, send, transcribeAudio, stop, diff --git a/webui/src/i18n/locales/en/common.json b/webui/src/i18n/locales/en/common.json index a31c8866c..524adb68b 100644 --- a/webui/src/i18n/locales/en/common.json +++ b/webui/src/i18n/locales/en/common.json @@ -198,6 +198,7 @@ "fileEditDisplay": "File edit display", "codeWrap": "Code wrapping", "brandLogos": "Brand logos", + "browserNotifications": "Task notifications", "maxResults": "Max results", "timeout": "Timeout", "jinaReader": "Jina reader", @@ -243,6 +244,7 @@ "fileEditDisplay": "Choose whether file edit activity opens as line counts or a diff.", "codeWrap": "Keep long code lines readable on smaller screens.", "brandLogos": "Show third-party provider and CLI logos in Settings.", + "browserNotifications": "Notify only when this page is in the background. Off by default.", "maxResults": "Results returned by each web_search call.", "timeout": "Seconds before a search provider request times out.", "jinaReader": "Use Jina Reader for web_fetch when available.", @@ -1013,7 +1015,8 @@ "activity": { "running": "Agent running", "complete": "Agent finished", - "updated": "New activity" + "updated": "New activity", + "recovery": "This conversation needs your attention" }, "pin": "Pin", "unpin": "Unpin", @@ -1436,6 +1439,18 @@ "estimated": "Includes estimated usage" } }, + "recovery": { + "actionFailed": "Recovery action failed. Try again.", + "interrupted": "Task interrupted", + "completed": "Task completed", + "failed": "Task recovery failed", + "failedHelp": "The saved task could not be restored safely. Review it before continuing.", + "resuming": "Restoring interrupted task…", + "review": "Review the task before continuing. Tools will not be replayed automatically.", + "safeResume": "Continuing from saved conversation context.", + "dismiss": "Dismiss", + "continue": "Continue" + }, "lightbox": { "title": "Image preview", "open": "View image", diff --git a/webui/src/i18n/locales/es/common.json b/webui/src/i18n/locales/es/common.json index 5dc215773..53c617bf1 100644 --- a/webui/src/i18n/locales/es/common.json +++ b/webui/src/i18n/locales/es/common.json @@ -161,6 +161,7 @@ "webuiDefaultAccess": "Acceso predeterminado", "currentModel": "Configuración actual", "brandLogos": "Logos de marca", + "browserNotifications": "Notificaciones de tareas", "cliAppsCatalog": "Catálogo", "cliAppsFilter": "Filtro", "engine": "Motor", @@ -204,6 +205,7 @@ "selectedModelProvider": "Definido por el modelo seleccionado.", "selectedModelValue": "Definido por el modelo seleccionado.", "brandLogos": "Muestra logos de proveedores de terceros y CLI en Ajustes.", + "browserNotifications": "Notifica solo cuando esta página está en segundo plano. Desactivado de forma predeterminada.", "cliAppsCatalog": "Instala solo adaptadores CLI de aplicaciones que nanobot puede ejecutar localmente; las aplicaciones nativas no se modifican.", "cliAppsFilter": "Busca por aplicación, categoría o capacidad.", "logs": "Abre la carpeta de registros del motor nativo.", @@ -1000,7 +1002,8 @@ "activity": { "running": "Agente en ejecución", "complete": "Agente terminado", - "updated": "Nueva actividad" + "updated": "Nueva actividad", + "recovery": "Esta conversación requiere tu atención" }, "pin": "Fijar", "unpin": "Desfijar", @@ -1423,6 +1426,18 @@ "automationSourceFallback": "Automatización", "automationTriggered": "Activada automáticamente" }, + "recovery": { + "actionFailed": "La recuperación falló. Inténtalo de nuevo.", + "interrupted": "Tarea interrumpida", + "completed": "Tarea completada", + "failed": "La recuperación de la tarea falló", + "failedHelp": "La tarea guardada no se pudo restaurar de forma segura. Revísala antes de continuar.", + "resuming": "Restaurando la tarea interrumpida…", + "review": "Revisa la tarea antes de continuar. Las herramientas no se repetirán automáticamente.", + "safeResume": "Continuando desde el contexto guardado de la conversación.", + "dismiss": "Descartar", + "continue": "Continuar" + }, "lightbox": { "title": "Vista previa de imagen", "open": "Ver imagen", diff --git a/webui/src/i18n/locales/fr/common.json b/webui/src/i18n/locales/fr/common.json index 360ebed75..245b87b5f 100644 --- a/webui/src/i18n/locales/fr/common.json +++ b/webui/src/i18n/locales/fr/common.json @@ -161,6 +161,7 @@ "webuiDefaultAccess": "Accès par défaut", "currentModel": "Configuration actuelle", "brandLogos": "Logos de marque", + "browserNotifications": "Notifications de tâches", "cliAppsCatalog": "Catalogue", "cliAppsFilter": "Filtre", "engine": "Moteur", @@ -204,6 +205,7 @@ "selectedModelProvider": "Défini par le modèle sélectionné.", "selectedModelValue": "Défini par le modèle sélectionné.", "brandLogos": "Affiche les logos de fournisseurs tiers et CLI dans les Réglages.", + "browserNotifications": "Notifier uniquement lorsque cette page est en arrière-plan. Désactivé par défaut.", "cliAppsCatalog": "Installe uniquement les adaptateurs CLI d’applications que nanobot peut exécuter localement ; les applications natives restent inchangées.", "cliAppsFilter": "Recherchez par application, catégorie ou capacité.", "logs": "Ouvre le dossier des journaux du moteur natif.", @@ -999,7 +1001,8 @@ "activity": { "running": "Agent en cours", "complete": "Agent terminé", - "updated": "Nouvelle activité" + "updated": "Nouvelle activité", + "recovery": "Cette conversation nécessite votre attention" }, "pin": "Épingler", "unpin": "Désépingler", @@ -1422,6 +1425,18 @@ "automationSourceFallback": "Automatisation", "automationTriggered": "Déclenché automatiquement" }, + "recovery": { + "actionFailed": "La récupération a échoué. Réessayez.", + "interrupted": "Tâche interrompue", + "completed": "Tâche terminée", + "failed": "Échec de la récupération de la tâche", + "failedHelp": "La tâche enregistrée n’a pas pu être restaurée en toute sécurité. Vérifiez-la avant de continuer.", + "resuming": "Restauration de la tâche interrompue…", + "review": "Vérifiez la tâche avant de continuer. Les outils ne seront pas relancés automatiquement.", + "safeResume": "Reprise depuis le contexte de conversation enregistré.", + "dismiss": "Ignorer", + "continue": "Continuer" + }, "lightbox": { "title": "Aperçu de l’image", "open": "Voir l’image", diff --git a/webui/src/i18n/locales/id/common.json b/webui/src/i18n/locales/id/common.json index fa4708f95..2c06bb75a 100644 --- a/webui/src/i18n/locales/id/common.json +++ b/webui/src/i18n/locales/id/common.json @@ -161,6 +161,7 @@ "webuiDefaultAccess": "Akses bawaan", "currentModel": "Konfigurasi saat ini", "brandLogos": "Logo merek", + "browserNotifications": "Notifikasi tugas", "cliAppsCatalog": "Katalog", "cliAppsFilter": "Saring", "engine": "Mesin", @@ -204,6 +205,7 @@ "selectedModelProvider": "Ditentukan oleh model yang dipilih.", "selectedModelValue": "Ditentukan oleh model yang dipilih.", "brandLogos": "Tampilkan logo penyedia pihak ketiga dan CLI di Pengaturan.", + "browserNotifications": "Beri tahu hanya saat halaman ini di latar belakang. Nonaktif secara bawaan.", "cliAppsCatalog": "Instal hanya adaptor CLI aplikasi yang dapat dijalankan nanobot secara lokal; aplikasi asli tidak diubah.", "cliAppsFilter": "Cari berdasarkan aplikasi, kategori, atau kemampuan.", "logs": "Buka folder log mesin asli.", @@ -999,7 +1001,8 @@ "activity": { "running": "Agen sedang berjalan", "complete": "Agen selesai", - "updated": "Aktivitas baru" + "updated": "Aktivitas baru", + "recovery": "Percakapan ini memerlukan perhatian Anda" }, "pin": "Sematkan", "unpin": "Lepas sematan", @@ -1422,6 +1425,18 @@ "automationSourceFallback": "Otomatisasi", "automationTriggered": "Dipicu otomatis" }, + "recovery": { + "actionFailed": "Pemulihan gagal. Coba lagi.", + "interrupted": "Tugas terputus", + "completed": "Tugas selesai", + "failed": "Pemulihan tugas gagal", + "failedHelp": "Tugas tersimpan tidak dapat dipulihkan dengan aman. Tinjau sebelum melanjutkan.", + "resuming": "Memulihkan tugas yang terputus…", + "review": "Tinjau tugas sebelum melanjutkan. Alat tidak akan dijalankan ulang secara otomatis.", + "safeResume": "Melanjutkan dari konteks percakapan yang tersimpan.", + "dismiss": "Abaikan", + "continue": "Lanjutkan" + }, "lightbox": { "title": "Pratinjau gambar", "open": "Lihat gambar", diff --git a/webui/src/i18n/locales/ja/common.json b/webui/src/i18n/locales/ja/common.json index 4622ca056..bc041a1d2 100644 --- a/webui/src/i18n/locales/ja/common.json +++ b/webui/src/i18n/locales/ja/common.json @@ -161,6 +161,7 @@ "webuiDefaultAccess": "既定の権限", "currentModel": "現在の設定", "brandLogos": "ブランドロゴ", + "browserNotifications": "タスク通知", "cliAppsCatalog": "カタログ", "cliAppsFilter": "フィルター", "engine": "エンジン", @@ -204,6 +205,7 @@ "selectedModelProvider": "選択したモデルによって設定されます。", "selectedModelValue": "選択したモデルによって設定されます。", "brandLogos": "設定で第三者プロバイダーと CLI のロゴを表示します。", + "browserNotifications": "このページがバックグラウンドにある場合のみ通知します。既定ではオフです。", "cliAppsCatalog": "nanobot がローカルで実行できるアプリ CLI アダプターだけをインストールします。ネイティブアプリは変更しません。", "cliAppsFilter": "アプリ、カテゴリ、機能で検索します。", "logs": "ネイティブエンジンのログフォルダーを開きます。", @@ -999,7 +1001,8 @@ "activity": { "running": "エージェント実行中", "complete": "エージェント完了", - "updated": "新しいアクティビティ" + "updated": "新しいアクティビティ", + "recovery": "この会話には対応が必要です" }, "pin": "ピン留め", "unpin": "ピン留めを解除", @@ -1422,6 +1425,18 @@ "automationSourceFallback": "自動化", "automationTriggered": "自動実行" }, + "recovery": { + "actionFailed": "復元操作に失敗しました。もう一度お試しください。", + "interrupted": "タスクが中断されました", + "completed": "タスクが完了しました", + "failed": "タスクの復元に失敗しました", + "failedHelp": "保存されたタスクを安全に復元できませんでした。続行前に確認してください。", + "resuming": "中断されたタスクを復元しています…", + "review": "続行する前にタスクを確認してください。ツールは自動的に再実行されません。", + "safeResume": "保存された会話コンテキストから続行しています。", + "dismiss": "閉じる", + "continue": "続行" + }, "lightbox": { "title": "画像プレビュー", "open": "画像を表示", diff --git a/webui/src/i18n/locales/ko/common.json b/webui/src/i18n/locales/ko/common.json index 7cff407ec..e33bdd925 100644 --- a/webui/src/i18n/locales/ko/common.json +++ b/webui/src/i18n/locales/ko/common.json @@ -161,6 +161,7 @@ "webuiDefaultAccess": "기본 권한", "currentModel": "현재 구성", "brandLogos": "브랜드 로고", + "browserNotifications": "작업 알림", "cliAppsCatalog": "카탈로그", "cliAppsFilter": "필터", "engine": "엔진", @@ -204,6 +205,7 @@ "selectedModelProvider": "선택한 모델에 의해 설정됩니다.", "selectedModelValue": "선택한 모델에 의해 설정됩니다.", "brandLogos": "설정에서 타사 제공자와 CLI 로고를 표시합니다.", + "browserNotifications": "이 페이지가 백그라운드에 있을 때만 알립니다. 기본값은 꺼짐입니다.", "cliAppsCatalog": "nanobot이 로컬에서 실행할 수 있는 앱 CLI 어댑터만 설치합니다. 네이티브 앱은 변경하지 않습니다.", "cliAppsFilter": "앱, 카테고리 또는 기능으로 검색합니다.", "logs": "네이티브 엔진 로그 폴더를 엽니다.", @@ -999,7 +1001,8 @@ "activity": { "running": "에이전트 실행 중", "complete": "에이전트 완료", - "updated": "새 활동" + "updated": "새 활동", + "recovery": "이 대화에는 확인이 필요합니다" }, "pin": "고정", "unpin": "고정 해제", @@ -1422,6 +1425,18 @@ "automationSourceFallback": "자동화", "automationTriggered": "자동 실행됨" }, + "recovery": { + "actionFailed": "복구 작업에 실패했습니다. 다시 시도하세요.", + "interrupted": "작업이 중단됨", + "completed": "작업 완료", + "failed": "작업 복구 실패", + "failedHelp": "저장된 작업을 안전하게 복구할 수 없습니다. 계속하기 전에 검토하세요.", + "resuming": "중단된 작업을 복구하는 중…", + "review": "계속하기 전에 작업을 검토하세요. 도구는 자동으로 다시 실행되지 않습니다.", + "safeResume": "저장된 대화 컨텍스트에서 계속합니다.", + "dismiss": "닫기", + "continue": "계속" + }, "lightbox": { "title": "이미지 미리보기", "open": "이미지 보기", diff --git a/webui/src/i18n/locales/pt-BR/common.json b/webui/src/i18n/locales/pt-BR/common.json index f6e0b0329..5525df084 100644 --- a/webui/src/i18n/locales/pt-BR/common.json +++ b/webui/src/i18n/locales/pt-BR/common.json @@ -198,6 +198,7 @@ "fileEditDisplay": "Exibição de edição de arquivo", "codeWrap": "Quebra de linha no código", "brandLogos": "Logos de marca", + "browserNotifications": "Notificações de tarefas", "maxResults": "Máx. de resultados", "timeout": "Tempo limite", "jinaReader": "Leitor Jina", @@ -243,6 +244,7 @@ "fileEditDisplay": "Escolha se a atividade de edição de arquivo é exibida como contagem de linhas ou como diferenças.", "codeWrap": "Mantém linhas longas de código legíveis em telas menores.", "brandLogos": "Mostra logotipos de provedores terceiros e de CLIs em Configurações.", + "browserNotifications": "Notifica somente quando esta página está em segundo plano. Desativado por padrão.", "maxResults": "Resultados retornados por cada chamada de web_search.", "timeout": "Segundos antes de uma requisição de busca expirar.", "jinaReader": "Usa o Jina Reader para web_fetch quando disponível.", @@ -1013,7 +1015,8 @@ "activity": { "running": "Agente em execução", "complete": "Agente finalizado", - "updated": "Nova atividade" + "updated": "Nova atividade", + "recovery": "Esta conversa precisa da sua atenção" }, "pin": "Fixar", "unpin": "Desafixar", @@ -1436,6 +1439,18 @@ "estimated": "Inclui uso estimado" } }, + "recovery": { + "actionFailed": "A recuperação falhou. Tente novamente.", + "interrupted": "Tarefa interrompida", + "completed": "Tarefa concluída", + "failed": "Falha ao recuperar a tarefa", + "failedHelp": "Não foi possível restaurar a tarefa salva com segurança. Revise-a antes de continuar.", + "resuming": "Restaurando a tarefa interrompida…", + "review": "Revise a tarefa antes de continuar. As ferramentas não serão executadas novamente de forma automática.", + "safeResume": "Continuando a partir do contexto de conversa salvo.", + "dismiss": "Descartar", + "continue": "Continuar" + }, "lightbox": { "title": "Pré-visualização de imagem", "open": "Ver imagem", diff --git a/webui/src/i18n/locales/vi/common.json b/webui/src/i18n/locales/vi/common.json index b17017947..f2a01bdda 100644 --- a/webui/src/i18n/locales/vi/common.json +++ b/webui/src/i18n/locales/vi/common.json @@ -161,6 +161,7 @@ "webuiDefaultAccess": "Quyền mặc định", "currentModel": "Cấu hình hiện tại", "brandLogos": "Logo thương hiệu", + "browserNotifications": "Thông báo tác vụ", "cliAppsCatalog": "Danh mục", "cliAppsFilter": "Bộ lọc", "engine": "Bộ máy", @@ -204,6 +205,7 @@ "selectedModelProvider": "Được đặt bởi mô hình đã chọn.", "selectedModelValue": "Được đặt bởi mô hình đã chọn.", "brandLogos": "Hiển thị logo nhà cung cấp bên thứ ba và CLI trong Cài đặt.", + "browserNotifications": "Chỉ thông báo khi trang này ở nền. Mặc định tắt.", "cliAppsCatalog": "Chỉ cài đặt các bộ chuyển đổi CLI ứng dụng mà nanobot có thể chạy cục bộ; ứng dụng gốc không bị thay đổi.", "cliAppsFilter": "Tìm theo ứng dụng, danh mục hoặc khả năng.", "logs": "Mở thư mục nhật ký của bộ máy gốc.", @@ -999,7 +1001,8 @@ "activity": { "running": "Tác nhân đang chạy", "complete": "Tác nhân đã hoàn tất", - "updated": "Hoạt động mới" + "updated": "Hoạt động mới", + "recovery": "Cuộc trò chuyện này cần bạn xử lý" }, "pin": "Ghim", "unpin": "Bỏ ghim", @@ -1422,6 +1425,18 @@ "automationSourceFallback": "Tự động hóa", "automationTriggered": "Tự động kích hoạt" }, + "recovery": { + "actionFailed": "Khôi phục thất bại. Hãy thử lại.", + "interrupted": "Tác vụ bị gián đoạn", + "completed": "Tác vụ đã hoàn tất", + "failed": "Khôi phục tác vụ thất bại", + "failedHelp": "Không thể khôi phục an toàn tác vụ đã lưu. Hãy xem lại trước khi tiếp tục.", + "resuming": "Đang khôi phục tác vụ bị gián đoạn…", + "review": "Hãy xem lại tác vụ trước khi tiếp tục. Công cụ sẽ không tự động chạy lại.", + "safeResume": "Đang tiếp tục từ ngữ cảnh hội thoại đã lưu.", + "dismiss": "Bỏ qua", + "continue": "Tiếp tục" + }, "lightbox": { "title": "Xem trước ảnh", "open": "Xem ảnh", diff --git a/webui/src/i18n/locales/zh-CN/common.json b/webui/src/i18n/locales/zh-CN/common.json index 93cc48ec6..a54857273 100644 --- a/webui/src/i18n/locales/zh-CN/common.json +++ b/webui/src/i18n/locales/zh-CN/common.json @@ -198,6 +198,7 @@ "fileEditDisplay": "文件编辑显示", "codeWrap": "代码换行", "brandLogos": "品牌 Logo", + "browserNotifications": "任务通知", "maxResults": "最大结果数", "timeout": "超时", "jinaReader": "Jina 阅读器", @@ -243,6 +244,7 @@ "fileEditDisplay": "选择文件编辑活动默认显示行数还是差异。", "codeWrap": "让长代码行在小屏幕上也易读。", "brandLogos": "在设置中显示第三方提供商和 CLI 图标。", + "browserNotifications": "仅在页面位于后台时通知，默认关闭。", "maxResults": "每次 web_search 调用返回的结果数。", "timeout": "搜索提供商请求超时前等待的秒数。", "jinaReader": "可用时为 web_fetch 使用 Jina Reader。", @@ -1013,7 +1015,8 @@ "activity": { "running": "智能体正在运行", "complete": "智能体已完成", - "updated": "有新内容" + "updated": "有新内容", + "recovery": "此对话需要你的处理" }, "pin": "置顶", "unpin": "取消置顶", @@ -1436,6 +1439,18 @@ "estimated": "包含估算用量" } }, + "recovery": { + "actionFailed": "恢复操作失败，请重试。", + "interrupted": "任务已中断", + "completed": "任务已完成", + "failed": "任务恢复失败", + "failedHelp": "无法安全恢复已保存的任务，继续前请先检查。", + "resuming": "正在恢复中断的任务…", + "review": "继续前请检查任务。工具不会被自动重放。", + "safeResume": "正在从已保存的对话上下文继续。", + "dismiss": "忽略", + "continue": "继续" + }, "lightbox": { "title": "图片预览", "open": "查看图片", diff --git a/webui/src/i18n/locales/zh-TW/common.json b/webui/src/i18n/locales/zh-TW/common.json index 99e05d33d..45fb1f39d 100644 --- a/webui/src/i18n/locales/zh-TW/common.json +++ b/webui/src/i18n/locales/zh-TW/common.json @@ -161,6 +161,7 @@ "webuiDefaultAccess": "預設存取權", "currentModel": "目前設定", "brandLogos": "品牌 Logo", + "browserNotifications": "任務通知", "cliAppsCatalog": "目錄", "cliAppsFilter": "篩選", "engine": "引擎", @@ -204,6 +205,7 @@ "selectedModelProvider": "由選取的模型決定。", "selectedModelValue": "由選取的模型決定。", "brandLogos": "在設定中顯示第三方供應商與 CLI 圖示。", + "browserNotifications": "僅在頁面位於背景時通知，預設關閉。", "cliAppsCatalog": "只安裝 nanobot 可在本機執行的應用程式專用 CLI 轉接器；不會改動原生應用程式。", "cliAppsFilter": "依應用程式、類別或功能搜尋。", "logs": "開啟原生引擎日誌資料夾。", @@ -999,7 +1001,8 @@ "activity": { "running": "智能體正在執行", "complete": "智能體已完成", - "updated": "有新內容" + "updated": "有新內容", + "recovery": "此對話需要你的處理" }, "pin": "置頂", "unpin": "取消置頂", @@ -1422,6 +1425,18 @@ "automationTriggered": "已自動觸發", "askAboutSelection": "繼續提問" }, + "recovery": { + "actionFailed": "復原操作失敗，請再試一次。", + "interrupted": "任務已中斷", + "completed": "任務已完成", + "failed": "任務復原失敗", + "failedHelp": "無法安全復原已儲存的任務，繼續前請先檢查。", + "resuming": "正在復原中斷的任務…", + "review": "繼續前請檢查任務。工具不會自動重播。", + "safeResume": "正在從已儲存的對話上下文繼續。", + "dismiss": "略過", + "continue": "繼續" + }, "lightbox": { "title": "圖片預覽", "open": "檢視圖片", diff --git a/webui/src/lib/api.ts b/webui/src/lib/api.ts index 62a754d2a..7af6fe3dd 100644 --- a/webui/src/lib/api.ts +++ b/webui/src/lib/api.ts @@ -22,6 +22,7 @@ import type { ProviderOAuthCompletionResult, ProviderOAuthLoginResult, ProviderSettingsUpdate, + RecoveryState, SessionDeleteResult, SessionHandle, SessionAutomationsPayload, @@ -192,6 +193,7 @@ export async function listSessions( preview?: string; model_preset?: string | null; run_started_at?: number | null; + recovery_state?: RecoveryState | null; workspace_scope?: WorkspaceScopePayload | null; handle?: SessionHandle | null; }; @@ -212,6 +214,7 @@ export async function listSessions( preview: s.preview ?? "", modelPreset: s.model_preset ?? null, runStartedAt: s.run_started_at ?? null, + recoveryState: s.recovery_state ?? null, workspaceScope: s.workspace_scope ?? null, handle, }; diff --git a/webui/src/lib/local-preferences.ts b/webui/src/lib/local-preferences.ts index 53e3c4836..67f42a2e2 100644 --- a/webui/src/lib/local-preferences.ts +++ b/webui/src/lib/local-preferences.ts @@ -7,6 +7,7 @@ export interface LocalPreferences { activityMode: LocalActivityMode; codeWrap: boolean; brandLogos: boolean; + browserNotifications: boolean; fileEditDisplayMode: FileEditDisplayMode; } @@ -18,6 +19,7 @@ export const DEFAULT_LOCAL_PREFS: LocalPreferences = { activityMode: "auto", codeWrap: true, brandLogos: false, + browserNotifications: false, fileEditDisplayMode: "summary", }; @@ -35,6 +37,7 @@ export function readLocalPreferences(): LocalPreferences { activityMode: parsed.activityMode === "expanded" ? "expanded" : "auto", codeWrap: parsed.codeWrap !== false, brandLogos: parsed.brandLogos === true, + browserNotifications: parsed.browserNotifications === true, fileEditDisplayMode: normalizeFileEditDisplayMode(parsed.fileEditDisplayMode), }; } catch { diff --git a/webui/src/lib/types.ts b/webui/src/lib/types.ts index dddfb38ae..d404d511e 100644 --- a/webui/src/lib/types.ts +++ b/webui/src/lib/types.ts @@ -49,6 +49,16 @@ export interface TurnUsage { [key: string]: number | undefined; } +export type RecoveryStatus = "resuming" | "awaiting_user" | "recovered" | "failed"; + +export interface RecoveryState { + status: RecoveryStatus; + recovery_id: string; + reason?: string; + attempts?: number; + can_continue?: boolean; +} + export interface UIMessage { id: string; role: Role; @@ -368,6 +378,8 @@ export interface ChatSummary { modelPreset?: string | null; /** Unix epoch seconds when this session currently has a turn in flight. */ runStartedAt?: number | null; + /** Durable recovery state that needs attention after an interrupted turn. */ + recoveryState?: RecoveryState | null; workspaceScope?: WorkspaceScopePayload | null; /** Stable, server-owned @handle for this session. */ handle?: SessionHandle | null; @@ -1246,6 +1258,7 @@ export type InboundEvent = event: "attached"; chat_id: string; temporary?: boolean; + recovery_state?: RecoveryState; usage?: TurnUsage; } | { @@ -1289,6 +1302,10 @@ export type InboundEvent = /** Optional structured payload on progress frames (channel-specific). */ agent_ui?: AgentUIBlob; } & InboundTurnMetadata) + | ({ + event: "recovery_state"; + chat_id: string; + } & RecoveryState) | ({ event: "file_edit"; chat_id: string; diff --git a/webui/src/tests/chat-list.test.tsx b/webui/src/tests/chat-list.test.tsx index 3ceacf094..0d4b95589 100644 --- a/webui/src/tests/chat-list.test.tsx +++ b/webui/src/tests/chat-list.test.tsx @@ -103,6 +103,24 @@ describe("ChatList", () => { ); }); + it("marks a conversation that needs recovery attention with a warning indicator", () => { + render( + , + ); + + expect(screen.getByRole("img", { name: "This conversation needs your attention" })) + .toBeInTheDocument(); + }); + it("keeps handle columns intact inside grouped panes", () => { render( { } }); + it("shows the running indicator while a recovery continuation is active", () => { + render( + , + ); + + expect(screen.getByRole("img", { name: "Agent running" })).toBeInTheDocument(); + expect(screen.queryByRole("img", { name: "This conversation needs your attention" })) + .not.toBeInTheDocument(); + }); + it("keeps tab grouping out of drag protocols while exposing inactive panes as mention sources", () => { render( { + beforeEach(() => localStorage.clear()); + + it("keeps browser notifications opt-in", () => { + expect(DEFAULT_LOCAL_PREFS.browserNotifications).toBe(false); + expect(readLocalPreferences().browserNotifications).toBe(false); + + writeLocalPreferences({ ...DEFAULT_LOCAL_PREFS, browserNotifications: true }); + expect(readLocalPreferences().browserNotifications).toBe(true); + }); +}); diff --git a/webui/src/tests/recovery-notice.test.tsx b/webui/src/tests/recovery-notice.test.tsx new file mode 100644 index 000000000..391f08ff3 --- /dev/null +++ b/webui/src/tests/recovery-notice.test.tsx @@ -0,0 +1,114 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import { RecoveryNotice } from "@/components/thread/RecoveryNotice"; + +const INTERRUPTED = { + status: "awaiting_user" as const, + recovery_id: "recovery-1", + reason: "tool_state_uncertain", +}; + +describe("RecoveryNotice", () => { + it("hides the internal resuming state after Continue is accepted", async () => { + const onContinue = vi.fn().mockResolvedValue(undefined); + + render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Continue" })); + + await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument()); + expect(onContinue).toHaveBeenCalledOnce(); + }); + + it("uses the shared status surface and motion treatment", () => { + render( + , + ); + + const notice = screen.getByRole("status"); + expect(notice).toHaveAttribute("data-recovery-status", "resuming"); + expect(notice).toHaveAttribute("aria-live", "polite"); + expect(notice).toHaveClass( + "max-w-[49.5rem]", + "rounded-control", + "animate-in", + "fade-in-0", + "slide-in-from-bottom-1", + "duration-200", + "motion-reduce:animate-none", + ); + }); + + it("keeps the notice visible when Continue fails", async () => { + const onContinue = vi.fn().mockRejectedValue(new Error("offline")); + + render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Continue" })); + + await waitFor(() => { + expect(screen.getByRole("alert")).toHaveTextContent("Recovery action failed"); + }); + }); + + it("shows the decision surface again when a continuation is interrupted", async () => { + const onContinue = vi.fn().mockResolvedValue(undefined); + const { rerender } = render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Continue" })); + await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument()); + + rerender( + , + ); + rerender( + , + ); + + await waitFor(() => expect(screen.getByRole("alert")).toBeInTheDocument()); + }); + + it("does not offer Continue when saved conversation context is unavailable", () => { + render( + , + ); + + expect(screen.queryByRole("button", { name: "Continue" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Dismiss" })).toBeInTheDocument(); + }); +}); diff --git a/webui/src/tests/thread-shell.test.tsx b/webui/src/tests/thread-shell.test.tsx index b2e5f5856..a244b729b 100644 --- a/webui/src/tests/thread-shell.test.tsx +++ b/webui/src/tests/thread-shell.test.tsx @@ -4007,4 +4007,32 @@ describe("ThreadShell", () => { expect(screen.getByRole("option", { name: /Other project/i })).toBeInTheDocument(); }); + it("allows a new turn after a completed recovery state", async () => { + const client = makeClient(); + render(wrap( + client, + {}} + />, + )); + + const input = await screen.findByLabelText("Message input"); + act(() => { + client._emitChat("recovered-chat", { + event: "recovery_state", + chat_id: "recovered-chat", + recovery_id: "recovery-1", + status: "recovered", + }); + }); + + fireEvent.change(input, { target: { value: "start the next task" } }); + fireEvent.click(screen.getByRole("button", { name: "Send message" })); + + expect(client.sendMessage).toHaveBeenCalledOnce(); + expect(screen.getByRole("button", { name: "Stop response" })).toBeInTheDocument(); + }); + }); diff --git a/webui/src/tests/useNanobotStream.test.tsx b/webui/src/tests/useNanobotStream.test.tsx index f609b8ca3..3ac2c0880 100644 --- a/webui/src/tests/useNanobotStream.test.tsx +++ b/webui/src/tests/useNanobotStream.test.tsx @@ -73,6 +73,7 @@ function fakeClient() { const runStartedAtByChatId = new Map(); const unsettledRunByChatId = new Map(); const goalStateByChatId = new Map(); + const requestMutation = vi.fn().mockResolvedValue({}); let status: ConnectionStatus = "open"; function recordGoalStatusForRunStrip(chatId: string, ev: InboundEvent) { @@ -133,6 +134,7 @@ function fakeClient() { return () => set!.delete(h); }, sendMessage: vi.fn(), + requestMutation, finishRunLocally: vi.fn(), newChat: vi.fn(), forkChat: vi.fn(), @@ -157,6 +159,7 @@ function fakeClient() { setUnsettled(chatId: string, unsettled: boolean) { unsettledRunByChatId.set(chatId, unsettled); }, + requestMutation, }; } @@ -395,6 +398,101 @@ describe("useNanobotStream", () => { }); }); + it("exposes typed recovery state and validates actions with its recovery id", async () => { + const fake = fakeClient(); + const { result } = renderHook(() => useNanobotStream("chat-recovery", EMPTY_MESSAGES), { + wrapper: wrap(fake.client), + }); + + act(() => { + fake.emit("chat-recovery", { + event: "goal_status", + chat_id: "chat-recovery", + status: "running", + started_at: 1_700, + }); + }); + expect(result.current.runStartedAt).toBe(1_700); + + act(() => { + fake.emit("chat-recovery", { + event: "recovery_state", + chat_id: "chat-recovery", + recovery_id: "recovery-1", + status: "awaiting_user", + reason: "tool_state_uncertain", + can_continue: false, + }); + }); + + expect(result.current.recoveryState).toEqual({ + recovery_id: "recovery-1", + status: "awaiting_user", + reason: "tool_state_uncertain", + can_continue: false, + }); + expect(result.current.isStreaming).toBe(false); + expect(result.current.runStartedAt).toBeNull(); + expect(fake.client.finishRunLocally).toHaveBeenCalledWith("chat-recovery"); + + act(() => { + fake.emit("chat-recovery", { + event: "recovery_state", + chat_id: "chat-recovery", + recovery_id: "recovery-1", + status: "awaiting_user", + reason: "tool_state_uncertain", + can_continue: true, + }); + }); + + await act(async () => result.current.continueRecovery()); + expect(fake.requestMutation).toHaveBeenCalledWith("recovery.continue", { + chat_id: "chat-recovery", + recovery_id: "recovery-1", + }); + + act(() => { + fake.emit("chat-recovery", { + event: "recovery_state", + chat_id: "chat-recovery", + recovery_id: "recovery-1", + status: "recovered", + }); + }); + expect(result.current.isStreaming).toBe(false); + expect(fake.client.finishRunLocally).toHaveBeenCalledWith("chat-recovery"); + }); + + it("does not let historical recovered state clear a later active turn", () => { + const fake = fakeClient(); + const { result } = renderHook( + () => useNanobotStream("chat-recovered-history", EMPTY_MESSAGES), + { wrapper: wrap(fake.client) }, + ); + + act(() => { + fake.emit("chat-recovered-history", { + event: "goal_status", + chat_id: "chat-recovered-history", + status: "running", + started_at: 1_700, + }); + fake.emit("chat-recovered-history", { + event: "attached", + chat_id: "chat-recovered-history", + recovery_state: { + recovery_id: "old-recovery", + status: "recovered", + }, + }); + }); + + expect(result.current.isStreaming).toBe(true); + expect(result.current.runStartedAt).toBe(1_700); + expect(fake.client.finishRunLocally).not.toHaveBeenCalled(); + }); + it("preserves proactive automation source metadata on complete assistant messages", () => { const fake = fakeClient(); const { result } = renderHook(() => useNanobotStream("chat-cron", EMPTY_MESSAGES), {