mirror/nanobot
1
0
Fork 0
mirror of https://github.com/HKUDS/nanobot.git synced 2026-08-31 00:03:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat(runtime): add user-controlled turn recovery

Browse Source
This commit is contained in:
Xubin Ren
2026-08-24 00:58:04 +08:00
parent ffa58aa5ef
commit 12029f8812
60 changed files with 4027 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nanobot/agent/loop.py
+103 -132
View File
@@ -79,6 +79,15 @@ from nanobot.session.model_selection import (
SESSION_MODEL_PRESET_METADATA_KEY,
model_preset_from_metadata,
)
from nanobot.session.recovery import (
PENDING_FOLLOWUP_ID_KEY,
RECOVERY_INBOUND_METADATA_KEY,
RecoveryAdmission,
acknowledge_pending_followups,
record_pending_followup,
restore_pending_interruption,
restore_runtime_checkpoint,
)
from nanobot.session.summary import SessionSummary
from nanobot.triggers.local_turns import LocalTriggerTurnCoordinator
from nanobot.utils.cancellation import task_is_cancelling
@@ -291,12 +300,14 @@ class AgentLoop:
restart_mode: str = "auto",
local_trigger_store: LocalTriggerStore | None = None,
idle_compact_check_interval_seconds: int = 0,
recovery_admission: RecoveryAdmission | None = None,
):
from nanobot.config.schema import ToolsConfig
_tc = tools_config or ToolsConfig()
defaults = AgentDefaults()
self.bus = bus
self._recovery_admission = recovery_admission
if turn_delivery_factory is not None:
if turn_delivery_factory.bus is not bus:
raise ValueError("turn delivery factory must use the agent message bus")
@@ -409,6 +420,7 @@ class AgentLoop:
# When a session has an active task, new messages for that session
# are routed here instead of creating a new task.
self._pending_queues: dict[str, asyncio.Queue[InboundMessage]] = {}
self._preserve_inflight_turns_on_shutdown = False
self._deferred_automation_turns: dict[str, list[InboundMessage]] = {}
self._cron_turns = CronTurnCoordinator(
publish_inbound=self.bus.publish_inbound,
@@ -726,6 +738,9 @@ class AgentLoop:
extra[RUNTIME_CONTEXT_HISTORY_META] = runtime_context_meta
session.add_message("user", text, **extra)
self._mark_pending_user_turn(session)
followup_id = msg.metadata.get(PENDING_FOLLOWUP_ID_KEY)
if isinstance(followup_id, str) and followup_id:
acknowledge_pending_followups(session, [followup_id])
self.sessions.save(session)
return True
return False
@@ -1061,6 +1076,9 @@ class AgentLoop:
row["subagent_task_id"] = task_id
row[HIDDEN_HISTORY_META] = subagent_marker
row["injected_event"] = "subagent_result"
followup_id = metadata.get(PENDING_FOLLOWUP_ID_KEY)
if isinstance(followup_id, str) and followup_id:
row[PENDING_FOLLOWUP_ID_KEY] = followup_id
return row
items: list[dict[str, Any]] = []
@@ -1285,6 +1303,17 @@ class AgentLoop:
break
if deferred:
continue
# A newer WebUI message must supersede an explicit recovery
# before it is injected into that recovery's pending queue.
# Without this admission point, a recovered turn could finish
# first and only then observe the user's newer request.
if (
effective_key in self._pending_queues
and msg.channel == "websocket"
and self._recovery_admission is not None
and not await self._recovery_admission.admit(msg)
):
continue
# If this session already has an active pending queue (i.e. a task
# is processing this session), route the message there for mid-turn
# injection instead of creating a competing task.
@@ -1303,6 +1332,17 @@ class AgentLoop:
msg,
session_key_override=effective_key,
)
session = self.sessions.get_or_create(effective_key)
followup_id = record_pending_followup(session, pending_msg)
if followup_id is not None:
pending_msg = dataclasses.replace(
pending_msg,
metadata={
**pending_msg.metadata,
PENDING_FOLLOWUP_ID_KEY: followup_id,
},
)
self.sessions.save(session)
try:
self._pending_queues[effective_key].put_nowait(pending_msg)
except asyncio.QueueFull:
@@ -1310,6 +1350,7 @@ class AgentLoop:
"Pending queue full for session {}, falling back to queued task",
effective_key,
)
msg = pending_msg
else:
logger.info(
"Routed follow-up message to pending queue for session {}",
@@ -1319,17 +1360,45 @@ class AgentLoop:
# Compute the effective session key before dispatching
# This ensures /stop command can find tasks correctly when unified session is enabled
task = asyncio.create_task(self._dispatch(msg))
active_tasks = self._active_tasks.setdefault(effective_key, set())
active_tasks: set[asyncio.Task[Any]] = self._active_tasks.setdefault(
effective_key,
set(),
)
active_tasks.add(task)
task.add_done_callback(active_tasks.discard)
finally:
await self.aclose()
def preserve_inflight_turns_on_shutdown(self) -> None:
"""Keep durable checkpoints when the owning gateway is restarting.
Normal cancellation intentionally materializes partial output so a
stopped gateway leaves a readable conversation. A managed restart is
different: RecoveryCoordinator needs the checkpoint intact to safely
offer the unfinished turn for explicit continuation after restart.
"""
self._preserve_inflight_turns_on_shutdown = True
async def _dispatch(self, msg: InboundMessage) -> None:
"""Process a message: per-session serial, cross-session concurrent."""
session_key = self._effective_session_key(msg)
if session_key != msg.session_key:
msg = dataclasses.replace(msg, session_key_override=session_key)
recovery_task_registered = False
recovery_admission = self._recovery_admission
current_task: asyncio.Task[Any] | None = None
if recovery_admission is not None:
recovery_id = msg.metadata.get(RECOVERY_INBOUND_METADATA_KEY)
if isinstance(recovery_id, str) and recovery_id:
current_task = asyncio.current_task()
if current_task is not None:
recovery_admission.register_recovery_task(session_key, current_task)
recovery_task_registered = True
if not await recovery_admission.admit(msg):
logger.info("Skipped stale recovery for session {}", session_key)
if recovery_task_registered and current_task is not None:
recovery_admission.unregister_recovery_task(session_key, current_task)
return
lock = self._get_session_lock(session_key)
gate = self._concurrency_gate or nullcontext()
@@ -1380,7 +1449,10 @@ class AgentLoop:
# _emit_checkpoint during tool execution; materializing
# it into session history now makes it visible in the
# next conversation turn.
if session_key in self._discarding_sessions:
if (
session_key in self._discarding_sessions
or self._preserve_inflight_turns_on_shutdown
):
raise
try:
key = self._effective_session_key(msg)
@@ -1437,6 +1509,12 @@ class AgentLoop:
await delivery.idle()
await self._publish_next_deferred_automation_turn(session_key)
finally:
if (
recovery_task_registered
and current_task is not None
and recovery_admission is not None
):
recovery_admission.unregister_recovery_task(session_key, current_task)
if pending is None:
await delivery.idle()
await self._publish_next_deferred_automation_turn(session_key)
@@ -1738,7 +1816,10 @@ class AgentLoop:
if self._restore_runtime_checkpoint(session):
self.sessions.save(session)
if self._restore_pending_user_turn(session):
if (
RECOVERY_INBOUND_METADATA_KEY not in msg.metadata
and restore_pending_interruption(session)
):
self.sessions.save(session)
async def _compact_session(self, ctx: TurnContext) -> None:
@@ -2093,8 +2174,21 @@ class AgentLoop:
if m.get("role") == "tool" and m.get("tool_call_id")
}
last_assistant_idx: int | None = None
saved_followup_ids: set[str] = set()
for m in messages[skip:]:
entry = dict(m)
followup_id_value = cast(object, entry.pop(PENDING_FOLLOWUP_ID_KEY, None))
followup_ids = (
[followup_id_value]
if isinstance(followup_id_value, str)
else [
followup_id
for followup_id in cast(list[object], followup_id_value)
if isinstance(followup_id, str)
]
if isinstance(followup_id_value, list)
else []
)
internal_meta = cast(object, entry.pop("_meta", None))
runtime_context_meta = (
cast(dict[str, Any], internal_meta).get(
@@ -2147,6 +2241,8 @@ class AgentLoop:
entry[RUNTIME_CONTEXT_HISTORY_META] = runtime_context_meta
entry.setdefault("timestamp", datetime.now().isoformat())
session.messages.append(entry)
if role == "user":
saved_followup_ids.update(followup_id for followup_id in followup_ids if followup_id)
if role == "assistant":
last_assistant_idx = len(session.messages) - 1
declared_tool_call_ids.update(
@@ -2161,6 +2257,8 @@ class AgentLoop:
)
if turn_latency_ms is not None and last_assistant_idx is not None:
session.messages[last_assistant_idx]["latency_ms"] = int(turn_latency_ms)
if saved_followup_ids:
acknowledge_pending_followups(session, saved_followup_ids)
session.updated_at = datetime.now()
def _persist_subagent_followup(self, session: Session, msg: InboundMessage) -> bool:
@@ -2195,7 +2293,7 @@ class AgentLoop:
def _set_runtime_checkpoint(self, session: Session, payload: dict[str, Any]) -> None:
"""Persist the latest in-flight turn state into session metadata."""
session.metadata[self._RUNTIME_CHECKPOINT_KEY] = payload
self.sessions.save(session)
self.sessions.save_runtime_checkpoint(session)
def _mark_pending_user_turn(self, session: Session) -> None:
session.metadata[self._PENDING_USER_TURN_KEY] = True
@@ -2207,136 +2305,9 @@ class AgentLoop:
if self._RUNTIME_CHECKPOINT_KEY in session.metadata:
session.metadata.pop(self._RUNTIME_CHECKPOINT_KEY, None)
@staticmethod
def _checkpoint_message_key(message: dict[str, Any]) -> tuple[Any, ...]:
return (
message.get("role"),
message.get("content"),
message.get("tool_call_id"),
message.get("name"),
message.get("tool_calls"),
message.get("reasoning_content"),
message.get("thinking_blocks"),
)
def _restore_runtime_checkpoint(self, session: Session) -> bool:
"""Materialize an unfinished turn into session history before a new request."""
from datetime import datetime
checkpoint = cast(
object,
session.metadata.get(self._RUNTIME_CHECKPOINT_KEY),
)
if not isinstance(checkpoint, dict):
return False
checkpoint_data = cast(dict[str, Any], checkpoint)
assistant_message = cast(object, checkpoint_data.get("assistant_message"))
completed_tool_results = cast(
Iterable[object],
checkpoint_data.get("completed_tool_results") or [],
)
pending_tool_calls = cast(
Iterable[object],
checkpoint_data.get("pending_tool_calls") or [],
)
restored_messages: list[dict[str, Any]] = []
if isinstance(assistant_message, dict):
restored = dict(cast(dict[str, Any], assistant_message))
restored.setdefault("timestamp", datetime.now().isoformat())
restored_messages.append(restored)
for message in completed_tool_results:
if isinstance(message, dict):
restored = dict(cast(dict[str, Any], message))
restored.setdefault("timestamp", datetime.now().isoformat())
restored_messages.append(restored)
for tool_call in pending_tool_calls:
if not isinstance(tool_call, dict):
continue
tool_call_data = cast(dict[str, Any], tool_call)
tool_id = tool_call_data.get("id")
function_data = cast(
dict[str, Any],
tool_call_data.get("function") or {},
)
name = function_data.get("name") or "tool"
restored_messages.append(
{
"role": "tool",
"tool_call_id": tool_id,
"name": name,
"content": "Error: Task interrupted before this tool finished.",
"timestamp": datetime.now().isoformat(),
}
)
overlap = 0
max_overlap = min(len(session.messages), len(restored_messages))
for size in range(max_overlap, 0, -1):
existing = session.messages[-size:]
restored = restored_messages[:size]
if all(
self._checkpoint_message_key(left) == self._checkpoint_message_key(right)
for left, right in zip(existing, restored)
):
overlap = size
break
appended_messages = restored_messages[overlap:]
session.messages.extend(appended_messages)
assistant_message_data = (
cast(dict[str, Any], assistant_message)
if isinstance(assistant_message, dict)
else None
)
provider_state_is_synchronized = (
checkpoint_data.get(self._PROVIDER_STATE_CHECKPOINT_VERSION_KEY)
== self._PROVIDER_STATE_CHECKPOINT_VERSION
)
phase = checkpoint_data.get("phase")
exact_final_response = (
phase == "final_response"
and assistant_message_data is not None
and assistant_message_data.get("role") == "assistant"
and not bool(checkpoint_data.get("completed_tool_results"))
and not bool(checkpoint_data.get("pending_tool_calls"))
)
exact_completed_tools = (
phase == "tools_completed"
and assistant_message_data is not None
and assistant_message_data.get("role") == "assistant"
and not bool(checkpoint_data.get("pending_tool_calls"))
)
if not (
provider_state_is_synchronized
and (exact_final_response or exact_completed_tools)
):
session.provider_state = None
self._clear_pending_user_turn(session)
self._clear_runtime_checkpoint(session)
return True
def _restore_pending_user_turn(self, session: Session) -> bool:
"""Close a turn that only persisted the user message before crashing."""
from datetime import datetime
if not session.metadata.get(self._PENDING_USER_TURN_KEY):
return False
if session.messages and session.messages[-1].get("role") == "user":
session.messages.append(
{
"role": "assistant",
"content": "Error: Task interrupted before a response was generated.",
"timestamp": datetime.now().isoformat(),
}
)
session.provider_state = None
session.updated_at = datetime.now()
self._clear_pending_user_turn(session)
return True
return restore_runtime_checkpoint(session)
async def process_direct(
self,
nanobot/agent/runner.py
+18
View File
@@ -37,6 +37,7 @@ from nanobot.runtime_context import (
reattach_runtime_context,
)
from nanobot.session.history_visibility import is_hidden_history_message
from nanobot.session.recovery import PENDING_FOLLOWUP_ID_KEY
from nanobot.utils.helpers import (
IncrementalThinkExtractor,
build_assistant_message,
@@ -234,6 +235,23 @@ class AgentRunner:
merged.get("content"),
injection.get("content"),
)
followup_id = injection.get(PENDING_FOLLOWUP_ID_KEY)
if isinstance(followup_id, str) and followup_id:
existing = cast(object, merged.get(PENDING_FOLLOWUP_ID_KEY))
followup_ids = (
[existing]
if isinstance(existing, str)
else [
item
for item in cast(list[object], existing)
if isinstance(item, str)
]
if isinstance(existing, list)
else []
)
if followup_id not in followup_ids:
followup_ids.append(followup_id)
merged[PENDING_FOLLOWUP_ID_KEY] = followup_ids
messages[-1] = merged
continue
messages.append(injection)
nanobot/bus/outbound_events.py
+9
View File
@@ -62,6 +62,15 @@ class TurnEndEvent(OutboundEvent):
context_window_tokens: int | None = None
@dataclass(frozen=True)
class RecoveryStateEvent(OutboundEvent):
status: str
recovery_id: str
reason: str | None = None
attempts: int = 0
can_continue: bool | None = None
@dataclass(frozen=True)
class GoalStatusEvent(OutboundEvent):
status: str
nanobot/channels/manager.py
+11
View File
@@ -104,6 +104,9 @@ class ChannelManager:
webui_mcp_runtime_status: Callable[[], Mapping[str, str]] | None = None,
webui_mcp_reload: Callable[[], Awaitable[dict[str, Any]]] | None = None,
webui_skill_state_action: Callable[[set[str]], None] | None = None,
webui_recovery_action: (
Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] | None
) = None,
config_path: Path | None = None,
):
if config_path is None:
@@ -126,6 +129,7 @@ class ChannelManager:
self._webui_mcp_runtime_status = webui_mcp_runtime_status
self._webui_mcp_reload = webui_mcp_reload
self._webui_skill_state_action = webui_skill_state_action
self._webui_recovery_action = webui_recovery_action
self.channels: dict[str, BaseChannel] = {}
self._channel_owners: dict[str, str] = {}
self._channel_runtime_specs: dict[str, tuple[str, str]] = {}
@@ -197,6 +201,7 @@ class ChannelManager:
mcp_runtime_status=self._webui_mcp_runtime_status,
mcp_reload=self._webui_mcp_reload,
skill_state_action=self._webui_skill_state_action,
recovery_action=self._webui_recovery_action,
logger=logger,
)
kwargs["gateway"] = gateway
@@ -615,6 +620,12 @@ class ChannelManager:
if target is None:
logger.warning("Restart notice target channel is not enabled: {}", notice.channel)
return
if notice.channel == "websocket":
# Reconnect and recovery are already represented by WebSocket
# protocol state. A generic restart-complete notice must not
# masquerade as a recovery transition and overwrite a real
# awaiting-user checkpoint in connected clients.
return
while not target.is_running:
remaining = deadline - loop.time()
nanobot/channels/websocket/runtime.py
+30
View File
@@ -32,6 +32,7 @@ from nanobot.bus.outbound_events import (
GoalStateSyncEvent,
GoalStatusEvent,
ProgressEvent,
RecoveryStateEvent,
RuntimeModelUpdatedEvent,
SessionUpdatedEvent,
TurnEndEvent,
@@ -55,6 +56,7 @@ from nanobot.security.workspace_access import (
)
from nanobot.session.goal_state import goal_state_ws_blob
from nanobot.session.model_selection import model_preset_from_metadata
from nanobot.session.recovery import recovery_state_from_metadata
from nanobot.session.webui_turns import (
clear_websocket_turn_if_current,
clear_websocket_turns,
@@ -453,6 +455,9 @@ class WebSocketChannel(BaseChannel):
self.logger.warning("ignoring invalid model preset metadata for chat_id={}", chat_id)
fields["model_preset"] = None
if isinstance(metadata, dict):
recovery_state = recovery_state_from_metadata(metadata)
if recovery_state is not None:
fields["recovery_state"] = recovery_state
usage = metadata.get("_last_usage")
if isinstance(usage, dict):
sanitized_usage: dict[str, int | float] = {}
@@ -1740,6 +1745,10 @@ class WebSocketChannel(BaseChannel):
provenance=event.provenance,
)
return
if isinstance(event, RecoveryStateEvent):
if conns:
await self.send_recovery_state(msg.chat_id, event)
return
if isinstance(event, GoalStateSyncEvent):
if conns:
await self.send_goal_state(msg.chat_id, event.goal_state or {"active": False})
@@ -2057,6 +2066,27 @@ class WebSocketChannel(BaseChannel):
for connection in conns:
await self._safe_send_to(connection, raw, label=" turn_end ")
async def send_recovery_state(
self,
chat_id: str,
event: RecoveryStateEvent,
) -> None:
"""Publish one structured recovery transition without chat pollution."""
body: dict[str, Any] = {
"event": "recovery_state",
"chat_id": chat_id,
"status": event.status,
"recovery_id": event.recovery_id,
"attempts": event.attempts,
}
if event.reason:
body["reason"] = event.reason
if event.can_continue is not None:
body["can_continue"] = event.can_continue
raw = json.dumps(body, ensure_ascii=False)
for connection in list(self._subs.get(chat_id, ())):
await self._safe_send_to(connection, raw, label=" recovery_state ")
async def send_goal_state(self, chat_id: str, blob: dict[str, Any]) -> None:
"""Push persisted goal-state snapshot for *chat_id* (multi-chat isolation)."""
conns = list(self._subs.get(chat_id, ()))
nanobot/channels/websocket/tests/test_websocket_channel.py
+34
View File
@@ -27,6 +27,7 @@ from nanobot.bus.outbound_events import (
GoalStateSyncEvent,
GoalStatusEvent,
ProgressEvent,
RecoveryStateEvent,
RuntimeModelUpdatedEvent,
SessionUpdatedEvent,
TurnEndEvent,
@@ -2720,6 +2721,39 @@ async def test_send_turn_end_emits_turn_end_event() -> None:
]
@pytest.mark.asyncio
async def test_recovery_state_is_a_structured_event_not_assistant_text() -> None:
bus = MagicMock()
channel = WebSocketChannel(
{"enabled": True, "allowFrom": ["*"]},
bus,
gateway=_basic_handler(bus),
)
mock_ws = AsyncMock()
channel._attach(mock_ws, "chat-1")
await channel.send(OutboundMessage(
channel="websocket",
chat_id="chat-1",
content="",
event=RecoveryStateEvent(
status="awaiting_user",
recovery_id="recovery-1",
reason="tool_state_unknown",
attempts=1,
),
))
assert _sent_ws_payloads(mock_ws) == [{
"event": "recovery_state",
"chat_id": "chat-1",
"status": "awaiting_user",
"recovery_id": "recovery-1",
"reason": "tool_state_unknown",
"attempts": 1,
}]
@pytest.mark.asyncio
async def test_system_command_turn_end_only_refreshes_session_metadata() -> None:
bus = MagicMock()
nanobot/channels/websocket/tests/test_websocket_http_routes.py
+26
View File
@@ -83,6 +83,7 @@ def _make_handler(
channel_feature_action: Any | None = None,
channel_runtime_status: Any | None = None,
mcp_reload: Any | None = None,
recovery_action: Any | None = None,
) -> GatewayServices:
config = WebSocketConfig.model_validate(cfg) if isinstance(cfg, dict) else cfg
workspace = workspace_path or Path.cwd()
@@ -103,6 +104,7 @@ def _make_handler(
channel_feature_action=channel_feature_action,
channel_runtime_status=channel_runtime_status,
mcp_reload=mcp_reload,
recovery_action=recovery_action,
)
@@ -121,6 +123,7 @@ def _ch(
channel_feature_action: Any | None = None,
channel_runtime_status: Any | None = None,
mcp_reload: Any | None = None,
recovery_action: Any | None = None,
**extra: Any,
) -> WebSocketChannel:
cfg: dict[str, Any] = {
@@ -145,6 +148,7 @@ def _ch(
channel_feature_action=channel_feature_action,
channel_runtime_status=channel_runtime_status,
mcp_reload=mcp_reload,
recovery_action=recovery_action,
)
return InProcessHttpChannel(cfg, bus, gateway=gateway)
@@ -3242,6 +3246,28 @@ async def _webui_mutate(
)
@pytest.mark.asyncio
async def test_recovery_mutation_uses_authenticated_websocket_action(bus: MagicMock) -> None:
recovery_action = AsyncMock(return_value={
"status": "resuming",
"recovery_id": "recovery-1",
})
channel = _ch(bus, recovery_action=recovery_action)
response = await _webui_mutate(
channel,
"recovery.continue",
{"chat_id": "chat-1", "recovery_id": "recovery-1"},
)
assert response.status_code == 200
assert response.json()["status"] == "resuming"
recovery_action.assert_awaited_once_with(
"continue",
{"chat_id": "chat-1", "recovery_id": "recovery-1"},
)
@pytest.mark.asyncio
async def test_workspace_folder_picker_is_local_authenticated_mutation(
bus: MagicMock,
nanobot/cli/gateway_runtime.py
+22
View File
@@ -322,6 +322,7 @@ def _run_gateway(
from nanobot.providers.fallback_provider import FallbackProvider
from nanobot.providers.image_generation import image_gen_provider_configs
from nanobot.session.manager import SessionManager
from nanobot.session.recovery import RecoveryCoordinator
from nanobot.session.webui_turns import (
WebuiTurnCoordinator,
WebuiTurnRoutePolicy,
@@ -422,6 +423,12 @@ def _run_gateway(
tools = ToolRegistry()
mcp_provider = MCPProvider.from_config(config, tools)
recovery = RecoveryCoordinator(
sessions=session_manager,
bus=bus,
unified_session=config.agents.defaults.unified_session,
)
# Create agent with cron service
agent = AgentLoop.from_config(
config, bus,
@@ -440,6 +447,7 @@ def _run_gateway(
local_trigger_store=trigger_store,
hook_factories=[create_file_edit_activity_hook],
tool_registry=tools,
recovery_admission=recovery,
)
def _schedule_webui_background(awaitable: Awaitable[None]) -> None:
agent.schedule_background(cast(Coroutine[Any, Any, None], awaitable))
@@ -448,6 +456,7 @@ def _run_gateway(
bus=bus,
sessions=session_manager,
schedule_background=_schedule_webui_background,
recovery=recovery,
)
webui_turn_coordinator.subscribe(runtime_events)
from nanobot.bus.events import OutboundMessage
@@ -683,6 +692,7 @@ def _run_gateway(
webui_mcp_runtime_status=mcp_provider.runtime_status,
webui_mcp_reload=mcp_provider.reload,
webui_skill_state_action=_webui_skill_state_action,
webui_recovery_action=recovery.handle_action,
config_path=Path(config_path),
)
@@ -849,6 +859,7 @@ def _run_gateway(
tasks: list[asyncio.Task[Any]] = []
shutdown_task: asyncio.Task[Any] | None = None
runtime_tasks: asyncio.Future[list[Any]] | None = None
startup_complete = False
shutdown_event = asyncio.Event()
cli_terminal._ensure_interactive_tty_mode()
restore_shutdown_handlers = _install_gateway_shutdown_handlers(
@@ -861,6 +872,10 @@ def _run_gateway(
await cron.start()
# Re-read once on first admission to close the watcher subscription window.
agent.runtime_resolver.invalidate()
# Recovery must finish before WebSocket and other channels begin
# accepting new input. That makes a new user message reliably
# supersede an old recoverable turn instead of racing its queue.
await recovery.scan()
async def _run_agent() -> None:
try:
await mcp_provider.connect()
@@ -915,6 +930,7 @@ def _run_gateway(
name="nanobot-webui-dev-server",
))
runtime_tasks = asyncio.gather(*tasks)
startup_complete = True
shutdown_task = asyncio.create_task(
shutdown_event.wait(),
name="nanobot-gateway-shutdown",
@@ -936,6 +952,10 @@ def _run_gateway(
console.print("\n[red]Error: Gateway crashed unexpectedly[/red]")
console.print(traceback.format_exc())
if not startup_complete:
# Do not report a successful gateway command when startup
# failed before any runtime task or listener was created.
raise typer.Exit(1)
finally:
try:
if shutdown_task and not shutdown_task.done():
@@ -943,6 +963,8 @@ def _run_gateway(
with suppress(asyncio.CancelledError):
await shutdown_task
cron.stop()
if gateway_runtime.preserves_inflight_turns_on_exit():
agent.preserve_inflight_turns_on_shutdown()
agent.stop()
# Cancel runtime tasks first, then deterministically close
# exec/MCP resources while the event loop is still alive.
nanobot/gateway/runtime.py
+55 -1
View File
@@ -201,6 +201,52 @@ class GatewayRuntime(ManagedProcessRuntime[ProcessStartOptions]):
"""Serialize long lifecycle transitions without blocking child cleanup."""
return FileLock(f"{self.paths.state_path}.transition.lock")
@property
def _restart_intent_path(self) -> Path:
"""Return the short-lived marker used to distinguish restart from stop.
A gateway receives the same operating-system termination request for a
graceful ``restart`` and an explicit ``stop``. The marker lets the
exiting process preserve its durable turn checkpoint only for the
former. It is intentionally local to one gateway instance.
"""
return self.paths.state_path.with_name(f"{self.paths.state_path.name}.restart")
def preserves_inflight_turns_on_exit(self) -> bool:
"""Whether this gateway was asked to exit as part of a managed restart."""
try:
raw_intent: object = json.loads(
self._restart_intent_path.read_text(encoding="utf-8")
)
except (json.JSONDecodeError, OSError):
return False
if not isinstance(raw_intent, dict):
return False
intent = cast(dict[str, object], raw_intent)
return intent.get("pid") == os.getpid()
def _write_restart_intent(self, pid: int) -> None:
self.paths.run_dir.mkdir(parents=True, exist_ok=True)
target = self._restart_intent_path
with tempfile.NamedTemporaryFile(
"w",
encoding="utf-8",
dir=target.parent,
prefix=f".{target.name}.",
delete=False,
) as handle:
json.dump({"pid": pid}, handle)
handle.flush()
os.fsync(handle.fileno())
temporary = Path(handle.name)
os.replace(temporary, target)
def _clear_restart_intent(self) -> None:
try:
self._restart_intent_path.unlink()
except FileNotFoundError:
pass
def start_background(self, options: ProcessStartOptions) -> RuntimeResult:
"""Start the gateway detached from the current terminal."""
lease = GatewayClientLease(self, kind="gateway-background")
@@ -353,7 +399,15 @@ class GatewayRuntime(ManagedProcessRuntime[ProcessStartOptions]):
"gateway_foreground_restart_required",
status,
)
stop_result = self._stop(timeout_s=timeout_s)
assert status.pid is not None
self._write_restart_intent(status.pid)
try:
stop_result = self._stop(timeout_s=timeout_s)
finally:
# The old process reads the marker while handling shutdown.
# Never let a stale marker turn a later explicit stop into a
# recoverable restart.
self._clear_restart_intent()
if not stop_result.ok:
return self._result(stop_result)
with self._lifecycle_lock():
nanobot/session/manager.py
+135 -2
View File
@@ -48,15 +48,21 @@ _SESSION_PREVIEW_MAX_CHARS = 120
_SESSION_LIST_PREVIEW_MAX_RECORDS = 200
_SESSION_LIST_PREVIEW_MAX_CHARS = 1_000_000
_SESSION_DATA_ERRORS = (ValueError, TypeError, AttributeError, KeyError)
_RUNTIME_CHECKPOINT_DATA_ERRORS = (OSError, *_SESSION_DATA_ERRORS)
_PROVIDER_STATE_RECORD_TYPE = "provider_state"
_PROVIDER_STATE_RECORD_PREFIX_RE = re.compile(
r'^\s*\{\s*"_type"\s*:\s*"provider_state"\s*(?:,|\})'
)
_RUNTIME_CHECKPOINT_KEY = "runtime_checkpoint"
_RUNTIME_CHECKPOINT_VERSION = 1
_RUNTIME_CHECKPOINT_SUFFIX = ".checkpoint.json"
_FORK_VOLATILE_METADATA_KEYS = {
"goal_state",
"pending_user_turn",
"pending_user_followups",
"runtime_checkpoint",
"session_handle",
"webui_recovery",
"thread_goal",
"title",
"title_user_edited",
@@ -1001,6 +1007,9 @@ class JsonlSessionStore:
def get_session_path(self, key: str) -> Path:
return self.sessions_dir / f"{self.storage_key(key)}.jsonl"
def get_runtime_checkpoint_path(self, key: str) -> Path:
return self.sessions_dir / f"{self.storage_key(key)}{_RUNTIME_CHECKPOINT_SUFFIX}"
def get_legacy_lossy_path(self, key: str) -> Path:
return self.sessions_dir / f"{safe_filename(key.replace(':', '_'))}.jsonl"
@@ -1066,7 +1075,7 @@ class JsonlSessionStore:
else:
messages.append(data)
return Session(
session = Session(
key=key,
messages=messages,
created_at=created_at or datetime.now(),
@@ -1075,6 +1084,8 @@ class JsonlSessionStore:
last_consolidated=last_consolidated,
provider_state=provider_state,
)
self._overlay_runtime_checkpoint_unlocked(session, path)
return session
except _SESSION_DATA_ERRORS as e:
logger.warning("Failed to load session {}: {}", key, e)
repaired = self._repair_unlocked(key)
@@ -1159,7 +1170,7 @@ class JsonlSessionStore:
if not messages and not metadata and provider_state is None:
return None
return Session(
session = Session(
key=key,
messages=messages,
created_at=created_at or datetime.now(),
@@ -1168,6 +1179,8 @@ class JsonlSessionStore:
last_consolidated=last_consolidated,
provider_state=provider_state,
)
self._overlay_runtime_checkpoint_unlocked(session, path)
return session
except _SESSION_DATA_ERRORS as e:
logger.warning("Repair failed for session {}: {}", key, e)
return None
@@ -1186,6 +1199,105 @@ class JsonlSessionStore:
with self._session_files_lock:
self._save_unlocked(session, fsync=fsync)
def save_runtime_checkpoint(self, session: Session) -> None:
"""Atomically persist only the volatile in-flight turn state.
A checkpoint is written several times during a tool-heavy turn. Keeping it
beside the append history avoids copying the full transcript at each safe
recovery boundary.
"""
with self._session_files_lock:
path = self.get_session_path(session.key)
if not path.exists():
# A user turn normally creates the session first. Internal callers
# may checkpoint a fresh session, so establish the durable base once.
self._save_unlocked(session)
return
checkpoint = session.metadata.get(_RUNTIME_CHECKPOINT_KEY)
if not isinstance(checkpoint, dict):
self.get_runtime_checkpoint_path(session.key).unlink(missing_ok=True)
return
payload: dict[str, Any] = {
"version": _RUNTIME_CHECKPOINT_VERSION,
"session_key": session.key,
"base_updated_at": session.updated_at.isoformat(),
"base_message_count": len(session.messages),
"checkpoint": checkpoint,
"provider_state": (
session.provider_state.to_private_record()
if session.provider_state is not None
else None
),
}
target = self.get_runtime_checkpoint_path(session.key)
tmp = target.with_name(f".{target.name}.{secrets.token_hex(8)}.tmp")
try:
with open(tmp, "x", encoding="utf-8") as handle:
os.chmod(tmp, 0o600)
json.dump(
payload,
handle,
ensure_ascii=False,
separators=(",", ":"),
)
os.replace(tmp, target)
finally:
tmp.unlink(missing_ok=True)
def _overlay_runtime_checkpoint_unlocked(self, session: Session, main_path: Path) -> None:
checkpoint_path = self.get_runtime_checkpoint_path(session.key)
try:
checkpoint_stat = checkpoint_path.lstat()
if not stat.S_ISREG(checkpoint_stat.st_mode):
logger.warning(
"Ignoring non-regular runtime checkpoint for session {}",
session.key,
)
return
# A complete session save supersedes an older sidecar. This comparison
# closes the small crash window between replacing the JSONL and unlinking
# its previous checkpoint.
if main_path.stat().st_mtime_ns > checkpoint_stat.st_mtime_ns:
checkpoint_path.unlink(missing_ok=True)
return
raw = _json_object(json.loads(checkpoint_path.read_text(encoding="utf-8")))
if (
raw.get("version") != _RUNTIME_CHECKPOINT_VERSION
or raw.get("session_key") != session.key
or raw.get("base_updated_at") != session.updated_at.isoformat()
or raw.get("base_message_count") != len(session.messages)
or not isinstance(raw.get("checkpoint"), dict)
):
checkpoint_path.unlink(missing_ok=True)
return
provider_record = raw.get("provider_state")
provider_state = (
None
if provider_record is None
else ProviderConversationState.from_private_record(provider_record)
)
if provider_record is not None and provider_state is None:
raise ValueError("invalid checkpoint provider state")
session.metadata[_RUNTIME_CHECKPOINT_KEY] = cast(
dict[str, Any], raw["checkpoint"]
)
session.provider_state = provider_state
except FileNotFoundError:
return
except _RUNTIME_CHECKPOINT_DATA_ERRORS as exc:
logger.warning(
"Ignoring invalid runtime checkpoint for session {}: {}",
session.key,
exc,
)
# Atomic writes mean a malformed target cannot become valid later.
# Remove it once so future loads do not repeatedly parse and log it.
with suppress(OSError):
if checkpoint_path.is_file() and not checkpoint_path.is_symlink():
checkpoint_path.unlink()
def _save_unlocked(self, session: Session, *, fsync: bool = False) -> None:
path = self.get_session_path(session.key)
tmp_path = path.with_name(f".{path.name}.{secrets.token_hex(8)}.tmp")
@@ -1215,6 +1327,10 @@ class JsonlSessionStore:
os.replace(tmp_path, path)
# The full record now contains the authoritative checkpoint state (or
# its removal), so an older volatile overlay is no longer needed.
self.get_runtime_checkpoint_path(session.key).unlink(missing_ok=True)
if fsync:
with suppress(PermissionError):
fd = os.open(str(path.parent), os.O_RDONLY)
@@ -1278,6 +1394,7 @@ class JsonlSessionStore:
def _delete_unlocked(self, key: str) -> bool:
paths = [
self.get_session_path(key),
self.get_runtime_checkpoint_path(key),
self.get_legacy_lossy_path(key),
self.get_legacy_session_path(key),
]
@@ -1585,6 +1702,10 @@ class SessionManager:
"""Get the collision-resistant workspace path for a session."""
return self._jsonl_store.get_session_path(key)
def _get_runtime_checkpoint_path(self, key: str) -> Path:
"""Get the private in-flight checkpoint path for a session."""
return self._jsonl_store.get_runtime_checkpoint_path(key)
def _get_legacy_lossy_path(self, key: str) -> Path:
"""Previous workspace session path using lossy ':' to '_' replacement."""
return self._jsonl_store.get_legacy_lossy_path(key)
@@ -1653,6 +1774,18 @@ class SessionManager:
self._store.save(session, fsync=fsync)
self._remember(session)
def save_runtime_checkpoint(self, session: Session) -> None:
"""Persist volatile recovery state without rewriting long history."""
if not session.policy.persist:
return
if self._store is self._jsonl_store:
self._jsonl_store.save_runtime_checkpoint(session)
self._remember(session)
return
# Third-party stores keep their existing all-or-nothing semantics until
# they opt into a dedicated checkpoint primitive.
self.save(session)
def rename_model_preset(self, old_name: str, new_name: str) -> int:
"""Rename a session-scoped model preset across durable and live sessions."""
if old_name == new_name:
nanobot/session/recovery.py
+911
View File
@@ -0,0 +1,911 @@
"""Durable, side-effect-safe recovery for interrupted WebUI turns.
The coordinator owns restart policy. AgentLoop only exposes checkpoint
materialization and an admission hook, so transport code never has to guess
whether an interrupted tool call is safe to replay.
"""
from __future__ import annotations
import asyncio
import dataclasses
import json
from collections.abc import Iterable, Mapping
from datetime import datetime
from typing import Any, Protocol, cast
from uuid import uuid4
from loguru import logger
from nanobot.bus.events import InboundMessage
from nanobot.bus.outbound_events import (
RecoveryStateEvent,
SessionUpdatedEvent,
outbound_message_for_event,
)
from nanobot.bus.queue import MessageBus
from nanobot.session import turn_continuation
from nanobot.session.keys import UNIFIED_SESSION_KEY, last_channel_from_metadata
from nanobot.session.manager import Session, SessionManager
from nanobot.webui.metadata import WEBUI_TURN_METADATA_KEY
RUNTIME_CHECKPOINT_KEY = "runtime_checkpoint"
PENDING_USER_TURN_KEY = "pending_user_turn"
RECOVERY_METADATA_KEY = "webui_recovery"
RECOVERY_INBOUND_METADATA_KEY = "_webui_recovery_id"
PENDING_FOLLOWUPS_KEY = "pending_user_followups"
PENDING_FOLLOWUP_ID_KEY = "_recovery_followup_id"
PROVIDER_STATE_CHECKPOINT_VERSION_KEY = "provider_state_checkpoint_version"
PROVIDER_STATE_CHECKPOINT_VERSION = "v1"
_RECOVERY_STATUSES = frozenset({"resuming", "awaiting_user", "recovered", "failed"})
_UNCERTAIN_TOOL_PHASES = frozenset({"awaiting_tools"})
_KNOWN_CHECKPOINT_PHASES = frozenset(
{"final_response", "tools_completed", "awaiting_tools", "error"}
)
class RecoveryActionError(ValueError):
"""A stale or malformed recovery action from an authenticated WebUI."""
def __init__(self, message: str, *, status: int = 400) -> None:
super().__init__(message)
self.status = status
class RecoveryAdmission(Protocol):
"""Narrow AgentLoop boundary for explicit recovery validation."""
async def admit(self, message: InboundMessage) -> bool: ...
def register_recovery_task(self, session_key: str, task: asyncio.Task[Any]) -> None: ...
def unregister_recovery_task(self, session_key: str, task: asyncio.Task[Any]) -> None: ...
def record_pending_followup(session: Session, message: InboundMessage) -> str | None:
"""Durably journal a WebUI follow-up before injecting it into a live turn."""
if message.channel != "websocket":
return None
try:
metadata_value: object = json.loads(json.dumps(message.metadata))
except (TypeError, ValueError):
logger.warning("Skipping non-serializable WebUI follow-up for recovery")
return None
if not isinstance(metadata_value, dict):
return None
metadata = cast(dict[str, Any], metadata_value)
existing_id = metadata.pop(PENDING_FOLLOWUP_ID_KEY, None)
followup_id = (
existing_id
if isinstance(existing_id, str) and existing_id
else uuid4().hex
)
records = _pending_followup_records(session)
if any(record.get("id") == followup_id for record in records):
return followup_id
records.append(
{
"id": followup_id,
"sender_id": message.sender_id,
"chat_id": message.chat_id,
"content": message.content,
"media": list(message.media or []),
"metadata": metadata,
}
)
# This journal is the recovery source of truth, not a mirror of the
# bounded in-memory injection queue. A queued turn can receive more
# follow-ups than the live queue accepts; dropping older journal entries
# would make those acknowledged user messages unrecoverable after a
# gateway restart. Entries are removed only once their user rows are
# committed by ``acknowledge_pending_followups``.
session.metadata[PENDING_FOLLOWUPS_KEY] = records
session.updated_at = datetime.now()
return followup_id
def pending_followups(session: Session) -> list[InboundMessage]:
"""Decode still-unacknowledged follow-ups from durable session metadata."""
messages: list[InboundMessage] = []
for record in _pending_followup_records(session):
followup_id = cast(object, record.get("id"))
sender_id = cast(object, record.get("sender_id"))
chat_id = cast(object, record.get("chat_id"))
content = cast(object, record.get("content"))
metadata = cast(object, record.get("metadata"))
if (
not isinstance(followup_id, str)
or not followup_id
or not isinstance(sender_id, str)
or not sender_id
or not isinstance(chat_id, str)
or not chat_id
):
continue
if not isinstance(content, str) or not isinstance(metadata, dict):
continue
media_value = cast(object, record.get("media"))
media = (
[item for item in cast(list[object], media_value) if isinstance(item, str)]
if isinstance(media_value, list)
else []
)
messages.append(
InboundMessage(
channel="websocket",
sender_id=sender_id,
chat_id=chat_id,
content=content,
media=media,
metadata={**cast(dict[str, Any], metadata), PENDING_FOLLOWUP_ID_KEY: followup_id},
session_key_override=session.key,
require_existing_session=True,
)
)
return messages
def acknowledge_pending_followups(session: Session, followup_ids: Iterable[str]) -> None:
"""Remove journal entries whose user rows were committed to history."""
acknowledged = set(followup_ids)
if not acknowledged:
return
records = [record for record in _pending_followup_records(session) if record.get("id") not in acknowledged]
if records:
session.metadata[PENDING_FOLLOWUPS_KEY] = records
else:
session.metadata.pop(PENDING_FOLLOWUPS_KEY, None)
def _pending_followup_records(session: Session) -> list[dict[str, Any]]:
raw = cast(object, session.metadata.get(PENDING_FOLLOWUPS_KEY))
if not isinstance(raw, list):
return []
values = cast(list[object], raw)
return [cast(dict[str, Any], value) for value in values if isinstance(value, dict)]
def _checkpoint_message_key(message: Mapping[str, Any]) -> tuple[Any, ...]:
return (
message.get("role"),
message.get("content"),
message.get("tool_call_id"),
message.get("name"),
message.get("tool_calls"),
message.get("reasoning_content"),
message.get("thinking_blocks"),
)
def _checkpoint_tool_call_ids(
value: object,
*,
result_rows: bool = False,
) -> list[str] | None:
"""Validate checkpoint tool rows and return their stable IDs."""
if not isinstance(value, list):
return None
ids: list[str] = []
for raw in cast(list[object], value):
if not isinstance(raw, dict):
return None
row = cast(dict[str, Any], raw)
id_key = "tool_call_id" if result_rows else "id"
call_id = cast(object, row.get(id_key))
if not isinstance(call_id, str) or not call_id:
return None
if result_rows:
if row.get("role") != "tool":
return None
else:
function_value = cast(object, row.get("function"))
if not isinstance(function_value, dict):
return None
function = cast(dict[str, Any], function_value)
name = cast(object, function.get("name"))
if not isinstance(name, str) or not name:
return None
ids.append(call_id)
return ids if len(ids) == len(set(ids)) else None
def _runtime_checkpoint_is_well_formed(checkpoint: Mapping[str, Any]) -> bool:
"""Return whether a checkpoint is safe to offer for continuation.
Restoration stays tolerant so Dismiss can always clear corrupt state.
Continue is stricter: silently dropping a malformed tool result could make
the model repeat an external side effect.
"""
assistant_value = cast(object, checkpoint.get("assistant_message"))
if not isinstance(assistant_value, dict):
return False
assistant = cast(dict[str, Any], assistant_value)
if assistant.get("role") != "assistant":
return False
completed_ids = _checkpoint_tool_call_ids(
cast(object, checkpoint.get("completed_tool_results")),
result_rows=True,
)
pending_ids = _checkpoint_tool_call_ids(
cast(object, checkpoint.get("pending_tool_calls")),
)
if completed_ids is None or pending_ids is None:
return False
assistant_calls_value = cast(object, assistant.get("tool_calls"))
assistant_call_ids = (
[]
if assistant_calls_value is None
else _checkpoint_tool_call_ids(assistant_calls_value)
)
if assistant_call_ids is None:
return False
phase = checkpoint.get("phase")
if phase == "final_response":
content = cast(object, assistant.get("content"))
return (
isinstance(content, str)
and bool(content.strip())
and not assistant_call_ids
and not completed_ids
and not pending_ids
)
if phase == "awaiting_tools":
return (
bool(assistant_call_ids)
and not completed_ids
and len(assistant_call_ids) == len(pending_ids)
and set(assistant_call_ids) == set(pending_ids)
)
if phase == "tools_completed":
return (
bool(assistant_call_ids)
and not pending_ids
and len(assistant_call_ids) == len(completed_ids)
and set(assistant_call_ids) == set(completed_ids)
)
# Error checkpoints have no current producer contract. Treat legacy or
# future instances as review-only until their exact persisted shape is
# specified; guessing here could make a partial side effect repeat.
return False
def restore_runtime_checkpoint(session: Session) -> bool:
"""Materialize the durable checkpoint exactly once and clear it.
Pending tool calls become explicit interrupted tool results. They are
never executed here. Provider-native state is retained only for the two
checkpoint shapes known to be synchronized with persisted history.
"""
checkpoint = cast(object, session.metadata.get(RUNTIME_CHECKPOINT_KEY))
if not isinstance(checkpoint, dict):
return False
data = cast(dict[str, Any], checkpoint)
assistant = cast(object, data.get("assistant_message"))
completed_value = cast(object, data.get("completed_tool_results"))
pending_value = cast(object, data.get("pending_tool_calls"))
completed = cast(list[object], completed_value) if isinstance(completed_value, list) else []
pending = cast(list[object], pending_value) if isinstance(pending_value, list) else []
restored: list[dict[str, Any]] = []
if isinstance(assistant, dict):
assistant_row = cast(dict[str, Any], assistant)
else:
assistant_row = {}
if assistant_row.get("role") == "assistant":
row = dict(assistant_row)
row.setdefault("timestamp", datetime.now().isoformat())
restored.append(row)
for value in completed:
if not isinstance(value, dict):
continue
tool_result = cast(dict[str, Any], value)
if tool_result.get("role") != "tool":
continue
row = dict(tool_result)
row.setdefault("timestamp", datetime.now().isoformat())
restored.append(row)
for value in pending:
if not isinstance(value, dict):
continue
tool_call = cast(dict[str, Any], value)
tool_call_id = tool_call.get("id")
function_value = cast(object, tool_call.get("function"))
if not isinstance(tool_call_id, str) or not tool_call_id:
continue
function = (
cast(dict[str, Any], function_value)
if isinstance(function_value, dict)
else {}
)
name = function.get("name")
restored.append(
{
"role": "tool",
"tool_call_id": tool_call_id,
"name": name if isinstance(name, str) and name else "tool",
"content": "Error: Task interrupted before this tool finished.",
"timestamp": datetime.now().isoformat(),
"_recovery_interrupted": True,
}
)
overlap = 0
for size in range(min(len(session.messages), len(restored)), 0, -1):
if all(
_checkpoint_message_key(left) == _checkpoint_message_key(right)
for left, right in zip(session.messages[-size:], restored[:size])
):
overlap = size
break
session.messages.extend(restored[overlap:])
assistant_data = cast(dict[str, Any], assistant) if isinstance(assistant, dict) else None
synchronized = (
data.get(PROVIDER_STATE_CHECKPOINT_VERSION_KEY)
== PROVIDER_STATE_CHECKPOINT_VERSION
)
phase = data.get("phase")
exact_final = (
phase == "final_response"
and assistant_data is not None
and assistant_data.get("role") == "assistant"
and not data.get("completed_tool_results")
and not data.get("pending_tool_calls")
)
exact_tools = (
phase == "tools_completed"
and assistant_data is not None
and assistant_data.get("role") == "assistant"
and not data.get("pending_tool_calls")
)
if not (synchronized and (exact_final or exact_tools)):
session.provider_state = None
session.metadata.pop(PENDING_USER_TURN_KEY, None)
session.metadata.pop(RUNTIME_CHECKPOINT_KEY, None)
session.updated_at = datetime.now()
return True
def _discard_runtime_checkpoint(session: Session) -> bool:
"""Drop checkpoint state that cannot be projected into valid history."""
if RUNTIME_CHECKPOINT_KEY not in session.metadata:
return False
session.metadata.pop(RUNTIME_CHECKPOINT_KEY, None)
session.provider_state = None
session.updated_at = datetime.now()
return True
def restore_pending_interruption(session: Session, *, superseded: bool = False) -> bool:
"""Close a persisted user-only turn without pretending it was answered."""
if not session.metadata.get(PENDING_USER_TURN_KEY):
return False
if session.messages and session.messages[-1].get("role") == "user":
content = (
"Task recovery was superseded by a newer message."
if superseded
else "Error: Task interrupted before a response was generated."
)
session.messages.append(
{
"role": "assistant",
"content": content,
"timestamp": datetime.now().isoformat(),
"_recovery_interrupted": True,
}
)
session.provider_state = None
session.updated_at = datetime.now()
session.metadata.pop(PENDING_USER_TURN_KEY, None)
return True
def append_recovery_interruption(session: Session, *, superseded: bool = False) -> None:
"""Close a restored partial turn whose last durable row is not the user message."""
if session.messages and session.messages[-1].get("_recovery_interrupted") is True:
return
session.messages.append(
{
"role": "assistant",
"content": (
"Task recovery was superseded by a newer message."
if superseded
else "Error: Task recovery was interrupted before completion."
),
"timestamp": datetime.now().isoformat(),
"_recovery_interrupted": True,
}
)
session.provider_state = None
session.updated_at = datetime.now()
def recovery_state_from_metadata(metadata: Mapping[str, Any] | None) -> dict[str, Any] | None:
"""Return a sanitized recovery state suitable for the WebSocket wire."""
value = metadata.get(RECOVERY_METADATA_KEY) if metadata else None
if not isinstance(value, dict):
return None
state = cast(dict[str, Any], value)
status = state.get("status")
recovery_id = state.get("recovery_id")
if status not in _RECOVERY_STATUSES or not isinstance(recovery_id, str):
return None
payload: dict[str, Any] = {"status": status, "recovery_id": recovery_id}
reason = state.get("reason")
if isinstance(reason, str) and reason:
payload["reason"] = reason
attempts = state.get("attempts")
if isinstance(attempts, int) and attempts >= 0:
payload["attempts"] = attempts
can_continue = state.get("can_continue")
if isinstance(can_continue, bool):
payload["can_continue"] = can_continue
return payload
@dataclasses.dataclass(slots=True)
class RecoveryCoordinator:
"""Classify, announce, and gate durable WebUI turn recovery."""
sessions: SessionManager
bus: MessageBus
unified_session: bool = False
_active_recovery_tasks: dict[str, asyncio.Task[Any]] = dataclasses.field(
default_factory=dict,
init=False,
repr=False,
)
def register_recovery_task(self, session_key: str, task: asyncio.Task[Any]) -> None:
"""Track the task that owns an explicit recovery continuation."""
self._active_recovery_tasks[session_key] = task
def unregister_recovery_task(self, session_key: str, task: asyncio.Task[Any]) -> None:
"""Drop a recovery task without removing a newer task for the same session."""
if self._active_recovery_tasks.get(session_key) is task:
self._active_recovery_tasks.pop(session_key, None)
async def _cancel_active_recovery(self, session_key: str) -> None:
"""Stop an explicit continuation before accepting newer user input."""
task = self._active_recovery_tasks.get(session_key)
if task is None or task is asyncio.current_task() or task.done():
return
task.cancel()
# AgentLoop's cancellation path materializes any partial checkpoint and
# releases its pending queue. Wait for that ownership to be released
# before the newer message is routed.
await asyncio.gather(task, return_exceptions=True)
async def scan(self) -> None:
"""Recover every interrupted WebUI session once at gateway startup."""
for key in self._recovery_candidates():
metadata_payload = self.sessions.read_session_metadata(key)
raw_metadata = metadata_payload.get("metadata") if metadata_payload else None
metadata = cast(dict[str, Any], raw_metadata) if isinstance(raw_metadata, dict) else {}
route = self._websocket_route_for(key, metadata)
if route is None:
continue
unfinished = self._has_unfinished_webui_transcript(key)
if not self._needs_recovery(metadata) and not unfinished:
continue
session = self.sessions.get_or_create(key)
try:
await self._recover_session(session, route[1])
await self._requeue_pending_followups(session)
except Exception:
logger.exception("failed to recover interrupted WebUI session {}", session.key)
state = recovery_state_from_metadata(session.metadata)
failed = self._set_state(
session,
status="failed",
recovery_id=cast(str, state["recovery_id"]) if state else uuid4().hex,
attempts=cast(int, state.get("attempts", 0)) if state else 0,
reason="recovery_failed",
can_continue=False,
)
self.sessions.save(session)
await self._publish(route[1], failed)
def _recovery_candidates(self) -> list[str]:
"""Discover canonical and transcript-only WebUI sessions cheaply."""
candidates = dict.fromkeys(
key
for item in self.sessions.list_sessions()
if isinstance((key := item.get("key")), str)
)
try:
# Imported lazily because the sidebar index also projects recovery
# metadata. The index is the owner of transcript-only discovery;
# duplicating its filename and migration rules here would drift.
from nanobot.webui.session_list_index import list_webui_sessions
for item in list_webui_sessions(self.sessions):
key = item.get("key")
if isinstance(key, str):
candidates.setdefault(key, None)
except Exception:
# Canonical checkpoint recovery remains available even if the
# optional display-history index is corrupt or unavailable.
logger.exception("failed to discover transcript-only WebUI sessions")
return list(candidates)
@staticmethod
def _needs_recovery(metadata: Mapping[str, Any]) -> bool:
if metadata.get(PENDING_USER_TURN_KEY) is True:
return True
if isinstance(metadata.get(RUNTIME_CHECKPOINT_KEY), dict):
return True
followups = metadata.get(PENDING_FOLLOWUPS_KEY)
if isinstance(followups, list) and len(cast(list[object], followups)) > 0:
return True
state = recovery_state_from_metadata(metadata)
return bool(state and state["status"] in {"resuming", "awaiting_user", "failed"})
async def admit(self, message: InboundMessage) -> bool:
"""Reject stale queued recoveries and let new user input supersede them."""
recovery_id = message.metadata.get(RECOVERY_INBOUND_METADATA_KEY)
if isinstance(recovery_id, str):
session = self.sessions.get_or_create(message.session_key)
state = recovery_state_from_metadata(session.metadata)
return bool(
state
and state["status"] == "resuming"
and state["recovery_id"] == recovery_id
)
if message.channel != "websocket":
return True
session = self.sessions.get_or_create(message.session_key)
state = recovery_state_from_metadata(session.metadata)
if state and state["status"] in {"resuming", "awaiting_user", "failed"}:
await self._cancel_active_recovery(message.session_key)
restore_runtime_checkpoint(session)
if not restore_pending_interruption(session, superseded=True):
append_recovery_interruption(session, superseded=True)
recovered = self._set_state(
session,
status="recovered",
recovery_id=cast(str, state["recovery_id"]),
attempts=cast(int, state.get("attempts", 0)),
reason="superseded",
)
self.sessions.save(session)
await self._publish(message.chat_id, recovered)
return True
async def turn_completed(self, session_key: str) -> None:
"""Resolve a resuming state after the recovered turn commits."""
session = self.sessions.get_or_create(session_key)
state = recovery_state_from_metadata(session.metadata)
if not state or state["status"] != "resuming":
return
route = self._websocket_route(session)
if route is None:
return
recovered = self._set_state(
session,
status="recovered",
recovery_id=cast(str, state["recovery_id"]),
attempts=cast(int, state.get("attempts", 0)),
reason="continued",
)
self.sessions.save(session)
await self._publish(route[1], recovered)
async def handle_action(self, action: str, payload: dict[str, Any]) -> dict[str, Any]:
"""Apply an authenticated continue/dismiss operation."""
chat_id = payload.get("chat_id")
recovery_id = payload.get("recovery_id")
if not isinstance(chat_id, str) or not chat_id:
raise RecoveryActionError("missing chat_id")
if not isinstance(recovery_id, str) or not recovery_id:
raise RecoveryActionError("missing recovery_id")
session = self.sessions.get_or_create(self._session_key(chat_id))
state = recovery_state_from_metadata(session.metadata)
if not state or state["recovery_id"] != recovery_id:
raise RecoveryActionError("recovery state is stale", status=409)
if action == "dismiss":
restore_runtime_checkpoint(session)
restore_pending_interruption(session)
next_state = self._set_state(
session,
status="recovered",
recovery_id=recovery_id,
attempts=cast(int, state.get("attempts", 0)),
reason="dismissed",
)
self.sessions.save(session)
await self._publish(chat_id, next_state)
return next_state
if action != "continue":
raise RecoveryActionError("unknown recovery action")
if state["status"] not in {"awaiting_user", "failed"}:
raise RecoveryActionError("recovery is not waiting for confirmation", status=409)
if state.get("can_continue") is False:
raise RecoveryActionError("recovery context is unavailable", status=409)
next_state = self._set_state(
session,
status="resuming",
recovery_id=recovery_id,
attempts=cast(int, state.get("attempts", 0)) + 1,
reason="user_confirmed",
resume_message_count=len(session.messages),
)
self.sessions.save(session)
await self._publish(chat_id, next_state)
await self._queue_continuation(session, chat_id, next_state)
return next_state
async def _recover_session(self, session: Session, chat_id: str) -> None:
checkpoint_value = cast(object, session.metadata.get(RUNTIME_CHECKPOINT_KEY))
checkpoint = (
cast(dict[str, Any], checkpoint_value)
if isinstance(checkpoint_value, dict)
else None
)
pending = session.metadata.get(PENDING_USER_TURN_KEY) is True
state = recovery_state_from_metadata(session.metadata)
if not pending and checkpoint is None:
if state and state["status"] == "resuming":
resume_count = self._resume_message_count(session)
if resume_count is not None and len(session.messages) > resume_count:
next_state = self._set_state(
session,
status="recovered",
recovery_id=cast(str, state["recovery_id"]),
attempts=cast(int, state.get("attempts", 0)),
reason="committed",
)
else:
next_state = self._set_state(
session,
status="awaiting_user",
recovery_id=cast(str, state["recovery_id"]),
attempts=cast(int, state.get("attempts", 1)),
reason="loop_guard",
)
self.sessions.save(session)
await self._publish(chat_id, next_state)
elif self._has_unfinished_webui_transcript(session.key):
# A normal last-client shutdown can materialize the checkpoint
# before the process exits. In that path there is no pending
# marker left to classify, but the append-only transcript still
# contains an activity row without a turn_end. Treat it as an
# interrupted turn instead of letting the UI resurrect it as a
# forever-running spinner.
waiting = self._set_state(
session,
status="awaiting_user",
recovery_id=uuid4().hex,
attempts=0,
reason="interrupted_without_checkpoint",
can_continue=False,
)
self.sessions.save(session)
await self._publish(chat_id, waiting)
return
if state and state["status"] in {"awaiting_user", "failed"}:
await self._publish(chat_id, state)
return
if state and state["status"] == "resuming":
restore_runtime_checkpoint(session)
restore_pending_interruption(session)
waiting = self._set_state(
session,
status="awaiting_user",
recovery_id=cast(str, state["recovery_id"]),
attempts=cast(int, state.get("attempts", 1)),
reason="loop_guard",
)
self.sessions.save(session)
await self._publish(chat_id, waiting)
return
recovery_id = uuid4().hex
phase = checkpoint.get("phase") if checkpoint is not None else None
pending_calls = checkpoint.get("pending_tool_calls") if checkpoint is not None else None
if checkpoint is not None and phase not in _KNOWN_CHECKPOINT_PHASES:
_discard_runtime_checkpoint(session)
restore_pending_interruption(session)
waiting = self._set_state(
session,
status="awaiting_user",
recovery_id=recovery_id,
attempts=0,
reason="checkpoint_unknown",
can_continue=False,
)
self.sessions.save(session)
await self._publish(chat_id, waiting)
return
if checkpoint is not None and not _runtime_checkpoint_is_well_formed(checkpoint):
_discard_runtime_checkpoint(session)
restore_pending_interruption(session)
waiting = self._set_state(
session,
status="awaiting_user",
recovery_id=recovery_id,
attempts=0,
reason="checkpoint_invalid",
can_continue=False,
)
self.sessions.save(session)
await self._publish(chat_id, waiting)
return
if phase == "final_response":
restore_runtime_checkpoint(session)
recovered = self._set_state(
session,
status="recovered",
recovery_id=recovery_id,
attempts=0,
reason="answer_restored",
)
self.sessions.save(session)
await self._publish(chat_id, recovered)
return
if phase in _UNCERTAIN_TOOL_PHASES or pending_calls:
restore_runtime_checkpoint(session)
waiting = self._set_state(
session,
status="awaiting_user",
recovery_id=recovery_id,
attempts=0,
reason="tool_state_unknown",
)
self.sessions.save(session)
await self._publish(chat_id, waiting)
return
# A gateway restart is a lifecycle boundary. Never enqueue model work
# implicitly: even a synchronized checkpoint may sit next to an
# external side effect that the user should review first. The final
# answer path above only restores persisted output; it never executes.
restore_runtime_checkpoint(session)
waiting = self._set_state(
session,
status="awaiting_user",
recovery_id=recovery_id,
attempts=0,
reason="restart_requires_confirmation",
)
self.sessions.save(session)
await self._publish(chat_id, waiting)
async def _queue_continuation(
self,
session: Session,
chat_id: str,
state: Mapping[str, Any],
) -> None:
recovery_id = cast(str, state["recovery_id"])
await self.bus.publish_inbound(
InboundMessage(
channel="websocket",
sender_id="system:recovery",
chat_id=chat_id,
content=(
"Continue the interrupted request from the saved conversation context. "
"Do not repeat completed work or mention the restart unless it affects the answer."
),
metadata={
"webui": True,
"_wants_stream": True,
WEBUI_TURN_METADATA_KEY: f"recovery:{recovery_id}",
RECOVERY_INBOUND_METADATA_KEY: recovery_id,
turn_continuation.INTERNAL_CONTINUATION_META: True,
turn_continuation.SKIP_USER_PERSIST_META: True,
},
session_key_override=session.key,
require_existing_session=True,
)
)
async def _requeue_pending_followups(self, session: Session) -> None:
"""Return durable live-turn follow-ups to the bus after a restart."""
for message in pending_followups(session):
await self.bus.publish_inbound(message)
@staticmethod
def _resume_message_count(session: Session) -> int | None:
raw_value = cast(object, session.metadata.get(RECOVERY_METADATA_KEY))
value = cast(dict[str, Any], raw_value) if isinstance(raw_value, dict) else None
if value is None:
return None
count = value.get("resume_message_count")
return count if isinstance(count, int) and count >= 0 else None
async def _publish(
self,
chat_id: str,
state: Mapping[str, Any],
) -> None:
"""Publish the recovery state and invalidate its sidebar projection."""
await self.bus.publish_outbound(
outbound_message_for_event(
channel="websocket",
chat_id=chat_id,
event=RecoveryStateEvent(
status=cast(str, state["status"]),
recovery_id=cast(str, state["recovery_id"]),
reason=cast(str | None, state.get("reason")),
attempts=cast(int, state.get("attempts", 0)),
can_continue=cast(bool | None, state.get("can_continue")),
),
)
)
await self.bus.publish_outbound(
outbound_message_for_event(
channel="websocket",
chat_id=chat_id,
event=SessionUpdatedEvent(scope="thread"),
)
)
@staticmethod
def _set_state(
session: Session,
*,
status: str,
recovery_id: str,
attempts: int,
reason: str,
resume_message_count: int | None = None,
can_continue: bool = True,
) -> dict[str, Any]:
state = {
"status": status,
"recovery_id": recovery_id,
"attempts": max(0, attempts),
"reason": reason,
"updated_at": datetime.now().isoformat(),
}
if not can_continue:
state["can_continue"] = False
if resume_message_count is not None:
state["resume_message_count"] = max(0, resume_message_count)
session.metadata[RECOVERY_METADATA_KEY] = state
session.updated_at = datetime.now()
return state
def _session_key(self, chat_id: str) -> str:
return UNIFIED_SESSION_KEY if self.unified_session else f"websocket:{chat_id}"
@staticmethod
def _has_unfinished_webui_transcript(session_key: str) -> bool:
"""Detect a stale WebUI activity tail after an unclean gateway stop.
The transcript is intentionally consulted only as a last-resort signal:
a durable pending turn or runtime checkpoint always takes precedence.
This keeps browser disconnects harmless while preventing a materialized
partial turn from being presented as active forever after a restart.
"""
try:
from nanobot.webui.transcript import has_unfinished_transcript_tail
return has_unfinished_transcript_tail(session_key)
except (OSError, ValueError, TypeError):
# Recovery must fail closed if the optional display transcript is
# corrupt or unavailable; the normal checkpoint path still applies.
return False
@staticmethod
def _websocket_route(session: Session) -> tuple[str, str] | None:
return RecoveryCoordinator._websocket_route_for(session.key, session.metadata)
@staticmethod
def _websocket_route_for(
session_key: str,
metadata: Mapping[str, Any],
) -> tuple[str, str] | None:
if session_key.startswith("websocket:"):
chat_id = session_key.split(":", 1)[1]
return ("websocket", chat_id) if chat_id else None
if session_key == UNIFIED_SESSION_KEY:
route = last_channel_from_metadata(metadata)
if route and route[0] == "websocket":
return route
return None
nanobot/session/webui_turns.py
+4
View File
@@ -43,6 +43,7 @@ from nanobot.runtime_context import public_history_message
from nanobot.session.goal_state import goal_state_ws_blob
from nanobot.session.history_visibility import is_hidden_history_message
from nanobot.session.manager import Session, SessionManager
from nanobot.session.recovery import RecoveryCoordinator
from nanobot.session.session_handles import session_handle_for_name
from nanobot.session.session_messages import (
SessionMessageEnvelope,
@@ -511,6 +512,7 @@ class WebuiTurnCoordinator:
bus: MessageBus
sessions: SessionManager
schedule_background: Callable[[Awaitable[None]], None]
recovery: RecoveryCoordinator | None = None
def subscribe(self, runtime_events: RuntimeEventBus) -> Callable[[], None]:
"""Subscribe this coordinator to runtime events."""
@@ -654,6 +656,8 @@ class WebuiTurnCoordinator:
event.runtime.context_window_tokens if event.runtime is not None else None
),
)
if self.recovery is not None:
await self.recovery.turn_completed(event.context.session_key)
self._schedule_title_update_from_event(event)
async def _handle_goal_state_changed(self, event: GoalStateChanged) -> None:
nanobot/webui/gateway_services.py
+2
View File
@@ -69,6 +69,7 @@ def build_gateway_services(
mcp_runtime_status: Callable[[], Mapping[str, str]] | None = None,
mcp_reload: Callable[[], Awaitable[dict[str, Any]]] | None = None,
skill_state_action: Callable[[set[str]], None] | None = None,
recovery_action: Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] | None = None,
logger: Any = default_logger,
) -> GatewayServices:
settings = WebUISettingsServices.create(
@@ -131,6 +132,7 @@ def build_gateway_services(
mcp_runtime_status=mcp_runtime_status,
mcp_reload=mcp_reload,
skill_state_action=skill_state_action,
recovery_action=recovery_action,
log=logger,
)
return GatewayServices(
nanobot/webui/session_list_index.py
+6 -1
View File
@@ -31,8 +31,9 @@ from nanobot.session.manager import (
_metadata_title, # pyright: ignore[reportPrivateUsage]
)
from nanobot.session.model_selection import model_preset_from_metadata
from nanobot.session.recovery import recovery_state_from_metadata
_INDEX_VERSION = 7
_INDEX_VERSION = 8
_INDEX_FILENAME = ".webui_session_index.json"
_MODEL_PRESET_FIELD = "model_preset"
_ROW_SOURCE_FIELD = "_source"
@@ -245,6 +246,7 @@ def _public_row(sessions_dir: Path, webui_dir: Path, row: dict[str, Any]) -> dic
"title": row.get("title", ""),
"preview": row.get("preview", ""),
_MODEL_PRESET_FIELD: row.get(_MODEL_PRESET_FIELD),
"recovery_state": row.get("recovery_state"),
_WORKSPACE_SCOPE_PRESENT_FIELD: row.get(_WORKSPACE_SCOPE_PRESENT_FIELD, False),
_WORKSPACE_SCOPE_VALUE_FIELD: row.get(_WORKSPACE_SCOPE_VALUE_FIELD),
"path": str(path),
@@ -485,6 +487,7 @@ def _indexed_row_for_session(session: Session, path: Path, webui_dir: Path) -> d
"title": _metadata_title(session.metadata),
"preview": _preview_from_messages(session.messages),
_MODEL_PRESET_FIELD: model_preset_from_metadata(session.metadata),
"recovery_state": recovery_state_from_metadata(session.metadata),
**_indexed_workspace_scope_fields(session.metadata),
_ROW_SOURCE_FIELD: _SESSION_SOURCE,
"file": path.name,
@@ -601,6 +604,7 @@ def _scan_transcript_row(
"title": "",
"preview": preview or fallback_preview,
_MODEL_PRESET_FIELD: None,
"recovery_state": None,
**_indexed_workspace_scope_fields({}),
_ROW_SOURCE_FIELD: _TRANSCRIPT_SOURCE,
"file": stem,
@@ -687,6 +691,7 @@ def _scan_session_row(
"title": _metadata_title(metadata),
"preview": preview or fallback_preview,
_MODEL_PRESET_FIELD: model_preset_from_metadata(metadata),
"recovery_state": recovery_state_from_metadata(metadata),
**_indexed_workspace_scope_fields(metadata),
_ROW_SOURCE_FIELD: _SESSION_SOURCE,
"file": path.name,
nanobot/webui/transcript.py
+13
View File
@@ -2541,6 +2541,19 @@ def has_pending_tool_calls(
return False
def has_unfinished_transcript_tail(session_key: str) -> bool:
"""Return whether the active transcript ends in an unfinished turn.
Recovery runs at gateway startup and only needs the newest, still-active
turn. Completed turns are rotated into immutable segment files, so reading
every historical segment here would make restart cost grow with the full
conversation history.
"""
return has_pending_tool_calls(
_read_transcript_file(webui_transcript_path(session_key))
)
def completed_turn_ids(lines: list[dict[str, Any]]) -> list[str]:
"""Return stable identities for turns with an explicitly persisted completion."""
completed: list[str] = []
nanobot/webui/ws_http.py
+39
View File
@@ -29,6 +29,7 @@ from nanobot.cron.session_turns import is_bound_cron_job
from nanobot.cron.types import CronJob, CronSchedule
from nanobot.security.workspace_access import WorkspaceScope
from nanobot.session.manager import SessionManager
from nanobot.session.recovery import RecoveryActionError
from nanobot.session.session_handles import (
SessionHandleResolver,
)
@@ -145,6 +146,8 @@ _WEBUI_MUTATION_PATHS = {
"skill.delete": "/api/webui/skills/delete",
"sidebar.update": "/api/webui/sidebar-state/update",
"workspace.pick_folder": "/api/workspaces/pick-folder",
"recovery.continue": "/api/webui/recovery/continue",
"recovery.dismiss": "/api/webui/recovery/dismiss",
"settings.agent.update": "/api/settings/update",
"settings.model_configuration.create": "/api/settings/model-configurations/create",
"settings.model_configuration.update": "/api/settings/model-configurations/update",
@@ -323,6 +326,9 @@ class GatewayHTTPHandler:
mcp_runtime_status: Callable[[], Mapping[str, str]] | None = None,
mcp_reload: Callable[[], Awaitable[dict[str, Any]]] | None = None,
skill_state_action: Callable[[set[str]], None] | None = None,
recovery_action: (
Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] | None
) = None,
log: Any = logger,
) -> None:
self.config = config
@@ -340,6 +346,7 @@ class GatewayHTTPHandler:
disabled_skills if disabled_skills is not None else set()
)
self.skill_state_action = skill_state_action
self.recovery_action = recovery_action
self._skill_install_lock = asyncio.Lock()
self._folder_picker_lock = asyncio.Lock()
self.cron_service = cron_service
@@ -454,6 +461,8 @@ class GatewayHTTPHandler:
return True
if re.match(r"^/api/webui/automations/(enable|disable|delete|run|update)$", path):
return True
if path in {"/api/webui/recovery/continue", "/api/webui/recovery/dismiss"}:
return True
return path in {
"/api/webui/skills/install",
"/api/webui/skills/update",
@@ -507,6 +516,11 @@ class GatewayHTTPHandler:
if response is not None:
return response
# Recovery routes
response = await self._dispatch_recovery_route(request, got)
if response is not None:
return response
# Session routes
response = await self._dispatch_session_routes(request, got)
if response is not None:
@@ -700,6 +714,27 @@ class GatewayHTTPHandler:
return None
async def _dispatch_recovery_route(
self,
request: WsRequest,
path: str,
) -> Response | None:
match = re.fullmatch(r"/api/webui/recovery/(continue|dismiss)", path)
if match is None:
return None
if not getattr(request, _WEBUI_MUTATION_REQUEST_ATTR, False):
return _http_error(405, "WebUI recovery actions require an authenticated WebSocket")
if self.recovery_action is None:
return _http_error(503, "WebUI recovery is unavailable")
payload = _mutation_payload(request)
if payload is None:
return _http_error(400, "invalid recovery payload")
try:
result = await self.recovery_action(match.group(1), payload)
except RecoveryActionError as exc:
return _http_error(exc.status, str(exc))
return _http_json_response(result)
async def _handle_session_context_get(self, request: WsRequest, key: str) -> Response:
if not self.check_api_token(request):
return _http_error(401, "Unauthorized")
@@ -746,6 +781,10 @@ class GatewayHTTPHandler:
for k, v in s.items()
if k != "path" and k not in WEBUI_SESSION_INDEX_INTERNAL_FIELDS
}
# Keep the additive recovery field absent for ordinary sessions so
# older clients and compact list responses stay unchanged.
if row.get("recovery_state") is None:
row.pop("recovery_state", None)
chat_id = key.split(":", 1)[1]
started_at = websocket_turn_wall_started_at(chat_id)
if started_at is not None:
tests/agent/test_loop_save_turn.py
+58
View File
@@ -36,6 +36,7 @@ from nanobot.session.keys import (
UNIFIED_SESSION_KEY,
)
from nanobot.session.manager import Session
from nanobot.session.recovery import PENDING_FOLLOWUP_ID_KEY, PENDING_FOLLOWUPS_KEY
from nanobot.session.turn_continuation import (
INTERNAL_CONTINUATION_META,
INTERNAL_CONTINUATION_RUN_STARTED_AT_META,
@@ -161,6 +162,35 @@ def test_persist_cron_turn_uses_distinct_history_marker(tmp_path: Path) -> None:
assert message["cron_prompt_ref"] == prompt_ref
def test_persist_user_message_acknowledges_durable_followup(tmp_path: Path) -> None:
loop = _make_full_loop(tmp_path)
session = loop.sessions.get_or_create("websocket:chat")
session.metadata[PENDING_FOLLOWUPS_KEY] = [
{
"id": "followup-1",
"sender_id": "user",
"chat_id": "chat",
"content": "queued while busy",
"media": [],
"metadata": {},
}
]
persisted = loop._persist_user_message_early(
InboundMessage(
channel="websocket",
sender_id="user",
chat_id="chat",
content="queued while busy",
metadata={PENDING_FOLLOWUP_ID_KEY: "followup-1"},
),
session,
)
assert persisted is True
assert PENDING_FOLLOWUPS_KEY not in session.metadata
def test_persist_local_trigger_turn_uses_hidden_automation_marker(tmp_path: Path) -> None:
loop = _make_full_loop(tmp_path)
session = loop.sessions.get_or_create("websocket:auto")
@@ -381,6 +411,34 @@ def test_save_turn_keeps_multimodal_runtime_context_for_model_replay() -> None:
assert public_history_message(session.messages[0])["content"] == []
def test_save_turn_acknowledges_every_merged_recovery_followup() -> None:
"""Persisting a merged injected row retires every durable follow-up ID."""
loop = _mk_loop()
session = Session(
key="test:recovery-followups",
metadata={
PENDING_FOLLOWUPS_KEY: [
{"id": "first"},
{"id": "second"},
]
},
)
loop._save_turn(
session,
[
{
"role": "user",
"content": "first\n\nsecond",
PENDING_FOLLOWUP_ID_KEY: ["first", "second"],
}
],
skip=0,
)
assert PENDING_FOLLOWUPS_KEY not in session.metadata
def test_save_turn_keeps_image_placeholder_and_runtime_context() -> None:
loop = _mk_loop()
session = Session(key="test:image")
tests/agent/test_runner_injections.py
+98 -2
View File
@@ -26,7 +26,7 @@ def _make_injection_callback(queue: asyncio.Queue):
return inject_cb
def _make_loop(tmp_path):
def _make_loop(tmp_path, *, recovery_admission=None):
from nanobot.agent.loop import AgentLoop
from nanobot.bus.queue import MessageBus
@@ -39,7 +39,12 @@ def _make_loop(tmp_path):
patch("nanobot.agent.loop.SubagentManager") as mock_sub_mgr:
mock_sub_mgr.return_value.cancel_by_session = AsyncMock(return_value=0)
mock_sub_mgr.return_value.close = AsyncMock()
loop = AgentLoop(bus=bus, provider=provider, workspace=tmp_path)
loop = AgentLoop(
bus=bus,
provider=provider,
workspace=tmp_path,
recovery_admission=recovery_admission,
)
return loop
@pytest.mark.asyncio
@@ -759,6 +764,20 @@ async def test_runner_merges_multiple_injected_user_messages_without_losing_medi
)
def test_runner_merge_keeps_all_recovery_followup_ids() -> None:
"""Merged follow-ups stay acknowledged together after a later save."""
from nanobot.agent.runner import AgentRunner
from nanobot.session.recovery import PENDING_FOLLOWUP_ID_KEY
messages = [{"role": "user", "content": "first", PENDING_FOLLOWUP_ID_KEY: "one"}]
AgentRunner._append_injected_messages(
messages,
[{"role": "user", "content": "second", PENDING_FOLLOWUP_ID_KEY: "two"}],
)
assert messages[-1][PENDING_FOLLOWUP_ID_KEY] == ["one", "two"]
def test_runner_merge_preserves_runtime_markers_with_media() -> None:
from nanobot.agent.runner import AgentRunner
from nanobot.runtime_context import (
@@ -967,6 +986,38 @@ async def test_followup_routed_to_pending_queue(tmp_path):
assert queued_msg.session_key == UNIFIED_SESSION_KEY
@pytest.mark.asyncio
async def test_websocket_followup_is_admitted_before_recovery_queue(tmp_path):
"""Recovery admission runs before a newer WebUI message is injected."""
from nanobot.bus.events import InboundMessage
admission = MagicMock()
admission.admit = AsyncMock(return_value=True)
loop = _make_loop(tmp_path, recovery_admission=admission)
loop._dispatch = AsyncMock() # type: ignore[method-assign]
session_key = "websocket:chat"
pending = asyncio.Queue(maxsize=20)
loop._pending_queues[session_key] = pending
run_task = asyncio.create_task(loop.run())
msg = InboundMessage(
channel="websocket",
sender_id="u",
chat_id="chat",
content="new request",
)
await loop.bus.publish_inbound(msg)
queued_msg = await asyncio.wait_for(pending.get(), timeout=2)
admission.admit.assert_awaited_once_with(msg)
assert queued_msg.content == msg.content
assert queued_msg.metadata["_recovery_followup_id"]
loop.stop()
await asyncio.wait_for(run_task, timeout=2)
@pytest.mark.asyncio
async def test_mid_turn_subagent_result_does_not_resolve_a_new_turn_route(tmp_path):
"""Injected results stay inside the active turn instead of opening a side turn."""
@@ -1314,6 +1365,51 @@ async def test_pending_queue_full_falls_back_to_queued_task(tmp_path):
assert pending.qsize() == 1
@pytest.mark.asyncio
async def test_pending_queue_overflow_keeps_websocket_followup_durable(tmp_path):
"""Fallback dispatch must not acknowledge a WebUI message before it commits."""
from nanobot.bus.events import InboundMessage
from nanobot.session.manager import Session
from nanobot.session.recovery import pending_followups
loop = _make_loop(tmp_path)
dispatched = asyncio.Event()
release_dispatch = asyncio.Event()
async def _dispatch(_msg):
dispatched.set()
await release_dispatch.wait()
loop._dispatch = AsyncMock(side_effect=_dispatch) # type: ignore[method-assign]
session = Session(key="websocket:c")
loop.sessions.get_or_create.return_value = session
pending = asyncio.Queue(maxsize=1)
pending.put_nowait(
InboundMessage(channel="websocket", sender_id="u", chat_id="c", content="already queued")
)
loop._pending_queues["websocket:c"] = pending
run_task = asyncio.create_task(loop.run())
await loop.bus.publish_inbound(
InboundMessage(
channel="websocket",
sender_id="u",
chat_id="c",
content="durable follow-up",
metadata={"webui": True},
)
)
await asyncio.wait_for(dispatched.wait(), timeout=2)
assert [message.content for message in pending_followups(session)] == ["durable follow-up"]
dispatched_msg = loop._dispatch.await_args.args[0]
assert dispatched_msg.metadata["_recovery_followup_id"]
release_dispatch.set()
loop.stop()
await asyncio.wait_for(run_task, timeout=2)
@pytest.mark.asyncio
async def test_dispatch_republishes_leftover_queue_messages(tmp_path):
"""Messages left in the pending queue after _dispatch are re-published to the bus.
tests/agent/test_stop_preserves_context.py
+22
View File
@@ -161,3 +161,25 @@ async def test_dispatch_cancellation_restores_checkpoint():
"Checkpoint metadata should be cleared after restore"
assert loop.sessions.save.called, \
"Session should be persisted so the restored state survives process restart"
@pytest.mark.asyncio
async def test_dispatch_cancellation_keeps_checkpoint_for_managed_restart(tmp_path: Path) -> None:
"""A restart preserves the checkpoint; an explicit stop still restores it."""
loop = _make_loop(tmp_path)
loop.preserve_inflight_turns_on_shutdown()
loop._restore_runtime_checkpoint = MagicMock() # type: ignore[method-assign]
async def _cancel(*_args: object, **_kwargs: object) -> None:
raise asyncio.CancelledError()
loop._process_message = _cancel # type: ignore[method-assign]
from nanobot.bus.events import InboundMessage
with pytest.raises(asyncio.CancelledError):
await loop._dispatch(
InboundMessage(channel="test", sender_id="u1", chat_id="c1", content="work")
)
loop._restore_runtime_checkpoint.assert_not_called()
tests/channels/test_channel_plugins.py
+21
View File
@@ -3721,6 +3721,27 @@ async def test_notify_restart_done_waits_until_channel_starts():
assert sent_msg.content.startswith("Restart completed")
@pytest.mark.asyncio
async def test_websocket_restart_notice_does_not_overwrite_recovery_state():
"""WebSocket attach/recovery events already own reconnect state."""
fake_config = SimpleNamespace(
channels=ChannelsConfig(),
providers=SimpleNamespace(groq=SimpleNamespace(api_key="")),
)
mgr = ChannelManager.__new__(ChannelManager)
mgr.config = fake_config
mgr.bus = MessageBus()
channel = _StartableChannel(fake_config, mgr.bus)
channel._running = True
mgr.channels = {"websocket": channel}
mgr._send_with_retry = AsyncMock()
notice = RestartNotice(channel="websocket", chat_id="chat", started_at_raw="100.0")
await mgr._send_restart_notice_when_started(notice)
mgr._send_with_retry.assert_not_awaited()
@pytest.mark.asyncio
async def test_restart_notice_retries_until_running_channel_accepts_delivery():
"""A running flag must not make an early transport failure final."""
tests/cli/test_commands.py
+17 -7
View File
@@ -103,6 +103,16 @@ class _GatewayAgentContractStub:
return None
class _EmptyGatewaySessionManager:
"""Minimal session-manager contract for gateway assembly tests."""
def list_sessions(self) -> list[dict[str, object]]:
return []
def flush_all(self) -> int:
return 0
def test_gateway_signal_handler_first_signal_stops_and_second_forces() -> None:
class _FakeLoop:
def __init__(self) -> None:
@@ -2756,7 +2766,7 @@ def _patch_serve_runtime(monkeypatch, config: Config, seen: dict[str, object]) -
monkeypatch,
config,
message_bus=lambda: object(),
session_manager=lambda _workspace: object(),
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
)
monkeypatch.setattr("nanobot.cli.commands.AgentLoop", _FakeAgentLoop)
monkeypatch.setattr("nanobot.api.server.create_app", _fake_create_app)
@@ -2823,7 +2833,7 @@ def test_gateway_uses_workspace_directory_for_cron_store(monkeypatch, tmp_path:
monkeypatch,
config,
message_bus=lambda: object(),
session_manager=lambda _workspace: object(),
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
cron_service=_StopCron,
)
@@ -3329,7 +3339,7 @@ def test_gateway_workspace_override_does_not_migrate_legacy_cron(
monkeypatch,
config,
message_bus=lambda: object(),
session_manager=lambda _workspace: object(),
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
cron_service=_StopCron,
get_cron_dir=lambda: legacy_dir,
)
@@ -3368,7 +3378,7 @@ def test_gateway_custom_config_workspace_does_not_migrate_legacy_cron(
monkeypatch,
config,
message_bus=lambda: object(),
session_manager=lambda _workspace: object(),
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
cron_service=_StopCron,
get_cron_dir=lambda: legacy_dir,
)
@@ -3569,7 +3579,7 @@ def test_gateway_health_endpoint_binds_and_serves_expected_responses(
monkeypatch,
config,
message_bus=lambda: object(),
session_manager=lambda _workspace: object(),
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
)
monkeypatch.setattr("nanobot.cli.gateway_runtime.AgentLoop", _FakeAgentLoop)
monkeypatch.setattr("nanobot.channels.manager.ChannelManager", _FakeChannelManager)
@@ -3771,7 +3781,7 @@ def test_gateway_agent_task_owns_initial_mcp_provider_close(
monkeypatch,
config,
message_bus=lambda: object(),
session_manager=lambda _workspace: object(),
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
)
monkeypatch.setattr("nanobot.cli.gateway_runtime.AgentLoop", _FakeAgentLoop)
monkeypatch.setattr("nanobot.cli.gateway_runtime.MCPProvider", _FakeMCPProvider)
@@ -3894,7 +3904,7 @@ def test_gateway_shutdown_event_exits_forever_runtime_tasks(
monkeypatch,
config,
message_bus=lambda: object(),
session_manager=lambda _workspace: object(),
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
)
monkeypatch.setattr("nanobot.cli.gateway_runtime.AgentLoop", _FakeAgentLoop)
monkeypatch.setattr("nanobot.channels.manager.ChannelManager", _FakeChannelManager)
tests/gateway/test_runtime.py
+43
View File
@@ -18,6 +18,7 @@ from nanobot.gateway import (
GatewayRuntimePaths,
GatewayStartOptions,
GatewayStatus,
RuntimeResult,
)
from nanobot.gateway.runtime import monitor_gateway_clients
from nanobot.process_runtime import process_is_running
@@ -454,6 +455,48 @@ def test_restart_does_not_detach_a_foreground_gateway(tmp_path, monkeypatch):
assert result.message == "gateway_foreground_restart_required"
def test_restart_marks_the_exiting_gateway_for_turn_recovery(tmp_path, monkeypatch):
"""A managed restart must not look like an explicit stop to the child."""
runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux")
status = GatewayStatus(
running=True,
pid=12345,
state_path=runtime.paths.state_path,
log_path=runtime.paths.log_path,
launch_mode="background",
)
monkeypatch.setattr(runtime, "status", lambda **_kwargs: status)
def stop(*, timeout_s: int):
assert timeout_s == 20
assert json.loads(runtime._restart_intent_path.read_text(encoding="utf-8")) == {
"pid": 12345
}
return SimpleNamespace(ok=True, message="gateway_stopped", status=status)
monkeypatch.setattr(runtime, "_stop", stop)
monkeypatch.setattr(
runtime,
"_start_background",
lambda _options: RuntimeResult(True, "gateway_started_background", status),
)
result = runtime.restart(GatewayStartOptions(port=18790))
assert result.ok is True
assert not runtime._restart_intent_path.exists()
def test_restart_intent_only_applies_to_the_recorded_gateway_process(tmp_path):
runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux")
runtime._write_restart_intent(os.getpid())
assert runtime.preserves_inflight_turns_on_exit() is True
runtime._write_restart_intent(os.getpid() + 1)
assert runtime.preserves_inflight_turns_on_exit() is False
def test_last_interactive_client_stops_an_on_demand_gateway(tmp_path, monkeypatch):
runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux")
monkeypatch.setattr(runtime, "_process_identity", lambda pid: pid)
tests/session/test_recovery.py
+732
View File
@@ -0,0 +1,732 @@
from __future__ import annotations
import asyncio
from pathlib import Path
import pytest
from nanobot.bus.events import InboundMessage
from nanobot.bus.outbound_events import RecoveryStateEvent, SessionUpdatedEvent
from nanobot.bus.queue import MessageBus
from nanobot.session.manager import Session, SessionManager
from nanobot.session.recovery import (
PENDING_FOLLOWUPS_KEY,
PENDING_USER_TURN_KEY,
RECOVERY_METADATA_KEY,
RUNTIME_CHECKPOINT_KEY,
RecoveryActionError,
RecoveryCoordinator,
acknowledge_pending_followups,
pending_followups,
record_pending_followup,
)
from nanobot.webui import session_list_index, transcript
def _persist(manager: SessionManager, session: Session) -> None:
session.metadata["webui"] = True
manager.save(session)
def _coordinator(workspace: Path) -> tuple[RecoveryCoordinator, MessageBus, SessionManager]:
bus = MessageBus()
sessions = SessionManager(workspace)
return RecoveryCoordinator(sessions, bus), bus, sessions
@pytest.mark.asyncio
async def test_restart_before_model_call_waits_for_confirmation(tmp_path: Path) -> None:
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "finish this"})
session.metadata[PENDING_USER_TURN_KEY] = True
_persist(sessions, session)
coordinator, bus, restarted = _coordinator(tmp_path)
await coordinator.scan()
assert bus.inbound.empty()
restored = restarted.get_or_create("websocket:chat")
assert restored.metadata[PENDING_USER_TURN_KEY] is True
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "restart_requires_confirmation"
@pytest.mark.asyncio
async def test_stale_incomplete_transcript_waits_for_confirmation(tmp_path: Path, monkeypatch) -> None:
"""A materialized shutdown must not reappear as an endless Working state."""
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
_persist(sessions, session)
monkeypatch.setattr(
"nanobot.webui.transcript.has_unfinished_transcript_tail",
lambda _key: True,
)
coordinator, bus, restarted = _coordinator(tmp_path)
await coordinator.scan()
assert bus.inbound.empty()
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
assert state["status"] == "awaiting_user"
assert state["reason"] == "interrupted_without_checkpoint"
event = bus.outbound.get_nowait().event
assert isinstance(event, RecoveryStateEvent)
assert event.status == "awaiting_user"
@pytest.mark.asyncio
async def test_transcript_only_interruption_is_discovered_without_materializing_completed_history(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
webui_dir = tmp_path / "webui"
webui_dir.mkdir()
monkeypatch.setattr(session_list_index, "get_webui_dir", lambda: webui_dir)
monkeypatch.setattr(transcript, "get_webui_dir", lambda: webui_dir)
unfinished_key = "websocket:unfinished"
completed_key = "websocket:completed"
(webui_dir / f"{SessionManager.safe_key(unfinished_key)}.jsonl").write_text(
'{"event":"user","chat_id":"unfinished","text":"keep going"}\n'
'{"event":"message","chat_id":"unfinished","kind":"progress","text":"Working"}\n',
encoding="utf-8",
)
(webui_dir / f"{SessionManager.safe_key(completed_key)}.jsonl").write_text(
'{"event":"user","chat_id":"completed","text":"done"}\n'
'{"event":"message","chat_id":"completed","text":"finished"}\n'
'{"event":"turn_end","chat_id":"completed"}\n',
encoding="utf-8",
)
coordinator, bus, sessions = _coordinator(tmp_path / "workspace")
await coordinator.scan()
restored = sessions.get_or_create(unfinished_key)
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "interrupted_without_checkpoint"
assert restored.metadata[RECOVERY_METADATA_KEY]["can_continue"] is False
assert sessions.read_session_metadata(completed_key) is None
event = bus.outbound.get_nowait().event
assert isinstance(event, RecoveryStateEvent)
assert event.status == "awaiting_user"
assert event.can_continue is False
assert bus.outbound.get_nowait().event.scope == "thread"
assert bus.outbound.empty()
with pytest.raises(RecoveryActionError, match="context is unavailable"):
await coordinator.handle_action(
"continue",
{"chat_id": "unfinished", "recovery_id": event.recovery_id},
)
@pytest.mark.asyncio
async def test_scan_loads_only_sessions_that_need_webui_recovery(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
sessions = SessionManager(tmp_path)
for key in ("telegram:idle", "discord:pending", "websocket:idle"):
session = sessions.get_or_create(key)
if key == "discord:pending":
session.metadata[PENDING_USER_TURN_KEY] = True
_persist(sessions, session)
pending = sessions.get_or_create("websocket:pending")
pending.metadata[PENDING_USER_TURN_KEY] = True
_persist(sessions, pending)
coordinator, _, restarted = _coordinator(tmp_path)
loaded: list[str] = []
get_or_create = restarted.get_or_create
def tracked_get_or_create(key: str) -> Session:
loaded.append(key)
return get_or_create(key)
monkeypatch.setattr(restarted, "get_or_create", tracked_get_or_create)
monkeypatch.setattr(
"nanobot.webui.transcript.has_unfinished_transcript_tail",
lambda _key: False,
)
await coordinator.scan()
assert loaded == ["websocket:pending"]
@pytest.mark.asyncio
async def test_live_turn_followup_survives_restart_until_it_is_committed(tmp_path: Path) -> None:
"""A message injected mid-turn is not lost between checkpoints."""
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
_persist(sessions, session)
followup_id = record_pending_followup(
session,
InboundMessage(
channel="websocket",
sender_id="user",
chat_id="chat",
content="also check the logs",
metadata={"webui": True},
),
)
assert followup_id is not None
sessions.save(session)
coordinator, bus, restarted = _coordinator(tmp_path)
await coordinator.scan()
queued = bus.inbound.get_nowait()
assert queued.content == "also check the logs"
assert queued.metadata["_recovery_followup_id"] == followup_id
restored = restarted.get_or_create("websocket:chat")
assert len(pending_followups(restored)) == 1
acknowledge_pending_followups(restored, [followup_id])
assert PENDING_FOLLOWUPS_KEY not in restored.metadata
def test_followup_journal_keeps_every_uncommitted_message(tmp_path: Path) -> None:
"""A live queue limit must never truncate durable WebUI follow-ups."""
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
followup_ids = [
record_pending_followup(
session,
InboundMessage(
channel="websocket",
sender_id="user",
chat_id="chat",
content=f"follow-up-{index}",
metadata={"webui": True},
),
)
for index in range(21)
]
assert all(followup_ids)
sessions.save(session)
restarted = SessionManager(tmp_path)
restored = restarted.get_or_create("websocket:chat")
assert [message.content for message in pending_followups(restored)] == [
f"follow-up-{index}" for index in range(21)
]
def test_requeued_followup_preserves_its_journal_id(tmp_path: Path) -> None:
"""Routing a recovered follow-up into a live turn must remain idempotent."""
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
original_id = record_pending_followup(
session,
InboundMessage(
channel="websocket",
sender_id="user",
chat_id="chat",
content="also check the logs",
metadata={"webui": True},
),
)
assert original_id is not None
recovered = pending_followups(session)[0]
assert record_pending_followup(session, recovered) == original_id
assert [record["id"] for record in session.metadata[PENDING_FOLLOWUPS_KEY]] == [
original_id
]
@pytest.mark.asyncio
async def test_completed_tools_wait_for_confirmation_after_restart(tmp_path: Path) -> None:
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "inspect"})
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
"phase": "tools_completed",
"assistant_message": {
"role": "assistant",
"content": "",
"tool_calls": [{"id": "call-1", "function": {"name": "read_file"}}],
},
"completed_tool_results": [
{
"role": "tool",
"tool_call_id": "call-1",
"name": "read_file",
"content": "saved result",
}
],
"pending_tool_calls": [],
}
_persist(sessions, session)
coordinator, bus, restarted = _coordinator(tmp_path)
await coordinator.scan()
assert bus.inbound.empty()
restored = restarted.get_or_create("websocket:chat")
assert restored.messages[-1]["content"] == "saved result"
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
@pytest.mark.asyncio
async def test_uncertain_tool_is_never_replayed(tmp_path: Path) -> None:
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "send it"})
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
"phase": "awaiting_tools",
"assistant_message": {
"role": "assistant",
"content": "",
"tool_calls": [{"id": "call-1", "function": {"name": "send_email"}}],
},
"completed_tool_results": [],
"pending_tool_calls": [
{"id": "call-1", "function": {"name": "send_email"}}
],
}
_persist(sessions, session)
coordinator, bus, restarted = _coordinator(tmp_path)
await coordinator.scan()
assert bus.inbound.empty()
restored = restarted.get_or_create("websocket:chat")
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "tool_state_unknown"
assert restored.messages[-1]["_recovery_interrupted"] is True
event = bus.outbound.get_nowait().event
assert isinstance(event, RecoveryStateEvent)
assert event.status == "awaiting_user"
@pytest.mark.asyncio
async def test_unknown_checkpoint_waits_for_confirmation(tmp_path: Path) -> None:
"""Malformed or newer checkpoint phases fail closed pending confirmation."""
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "deploy it"})
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {"phase": "future_phase"}
_persist(sessions, session)
coordinator, bus, restarted = _coordinator(tmp_path)
await coordinator.scan()
assert bus.inbound.empty()
restored = restarted.get_or_create("websocket:chat")
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "checkpoint_unknown"
assert restored.metadata[RECOVERY_METADATA_KEY]["can_continue"] is False
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
assert PENDING_USER_TURN_KEY not in restored.metadata
assert [message["role"] for message in restored.messages] == ["user", "assistant"]
assert restored.messages[-1]["_recovery_interrupted"] is True
with pytest.raises(RecoveryActionError, match="context is unavailable"):
await coordinator.handle_action(
"continue",
{
"chat_id": "chat",
"recovery_id": restored.metadata[RECOVERY_METADATA_KEY]["recovery_id"],
},
)
dismissed = await coordinator.handle_action(
"dismiss",
{
"chat_id": "chat",
"recovery_id": restored.metadata[RECOVERY_METADATA_KEY]["recovery_id"],
},
)
assert dismissed["status"] == "recovered"
@pytest.mark.asyncio
async def test_malformed_checkpoint_can_always_be_dismissed(tmp_path: Path) -> None:
"""Corrupt private state must not trap the user in a failed recovery notice."""
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "deploy it"})
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
"phase": "future_phase",
"assistant_message": "invalid",
"completed_tool_results": 3,
"pending_tool_calls": [{"id": "call-1", "function": "invalid"}],
}
_persist(sessions, session)
coordinator, _, restarted = _coordinator(tmp_path)
await coordinator.scan()
restored = restarted.get_or_create("websocket:chat")
state = restored.metadata[RECOVERY_METADATA_KEY]
result = await coordinator.handle_action(
"dismiss",
{"chat_id": "chat", "recovery_id": state["recovery_id"]},
)
assert result["status"] == "recovered"
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
@pytest.mark.asyncio
async def test_known_but_malformed_checkpoint_cannot_continue(tmp_path: Path) -> None:
"""A known phase does not make corrupt tool state safe to resume."""
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "send it"})
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
"phase": "tools_completed",
"assistant_message": {"role": "assistant", "content": "working"},
"completed_tool_results": "missing durable results",
"pending_tool_calls": [],
}
_persist(sessions, session)
coordinator, _, restarted = _coordinator(tmp_path)
await coordinator.scan()
restored = restarted.get_or_create("websocket:chat")
state = restored.metadata[RECOVERY_METADATA_KEY]
assert state["status"] == "awaiting_user"
assert state["reason"] == "checkpoint_invalid"
assert state["can_continue"] is False
with pytest.raises(RecoveryActionError, match="context is unavailable"):
await coordinator.handle_action(
"continue",
{"chat_id": "chat", "recovery_id": state["recovery_id"]},
)
@pytest.mark.asyncio
async def test_malformed_final_response_is_not_reported_as_restored(tmp_path: Path) -> None:
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "answer"})
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
"phase": "final_response",
"assistant_message": "not an answer row",
"completed_tool_results": [],
"pending_tool_calls": [],
}
_persist(sessions, session)
coordinator, _, restarted = _coordinator(tmp_path)
await coordinator.scan()
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
assert state["status"] == "awaiting_user"
assert state["reason"] == "checkpoint_invalid"
assert state["can_continue"] is False
restored = restarted.get_or_create("websocket:chat")
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
assert PENDING_USER_TURN_KEY not in restored.metadata
assert [message["role"] for message in restored.messages] == ["user", "assistant"]
assert restored.messages[-1]["_recovery_interrupted"] is True
assert all("tool_calls" not in message for message in restored.messages)
@pytest.mark.asyncio
async def test_checkpoint_with_missing_tool_result_cannot_continue(tmp_path: Path) -> None:
"""Never resume when persisted results do not cover every requested tool."""
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "send both"})
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
"phase": "tools_completed",
"assistant_message": {
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "call-1", "function": {"name": "send_email"}},
{"id": "call-2", "function": {"name": "send_email"}},
],
},
"completed_tool_results": [
{
"role": "tool",
"tool_call_id": "call-1",
"name": "send_email",
"content": "sent",
}
],
"pending_tool_calls": [],
}
_persist(sessions, session)
coordinator, _, restarted = _coordinator(tmp_path)
await coordinator.scan()
restored = restarted.get_or_create("websocket:chat")
state = restored.metadata[RECOVERY_METADATA_KEY]
assert state["status"] == "awaiting_user"
assert state["reason"] == "checkpoint_invalid"
assert state["can_continue"] is False
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
assert PENDING_USER_TURN_KEY not in restored.metadata
assert [message["role"] for message in restored.messages] == ["user", "assistant"]
assert restored.messages[-1]["_recovery_interrupted"] is True
assert all("tool_calls" not in message for message in restored.messages)
@pytest.mark.asyncio
async def test_empty_final_response_is_not_reported_as_restored(tmp_path: Path) -> None:
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "answer"})
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
"phase": "final_response",
"assistant_message": {"role": "assistant", "content": ""},
"completed_tool_results": [],
"pending_tool_calls": [],
}
_persist(sessions, session)
coordinator, _, restarted = _coordinator(tmp_path)
await coordinator.scan()
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
assert state["status"] == "awaiting_user"
assert state["reason"] == "checkpoint_invalid"
assert state["can_continue"] is False
@pytest.mark.asyncio
async def test_final_answer_is_restored_without_model_call(tmp_path: Path) -> None:
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "answer"})
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
"phase": "final_response",
"assistant_message": {"role": "assistant", "content": "already finished"},
"completed_tool_results": [],
"pending_tool_calls": [],
}
_persist(sessions, session)
coordinator, bus, restarted = _coordinator(tmp_path)
await coordinator.scan()
assert bus.inbound.empty()
restored = restarted.get_or_create("websocket:chat")
assert restored.messages[-1]["content"] == "already finished"
first = bus.outbound.get_nowait().event
second = bus.outbound.get_nowait().event
assert isinstance(first, RecoveryStateEvent) and first.status == "recovered"
assert isinstance(second, SessionUpdatedEvent)
@pytest.mark.asyncio
async def test_explicit_recovery_continue_queues_once(tmp_path: Path) -> None:
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "continue"})
session.metadata[PENDING_USER_TURN_KEY] = True
_persist(sessions, session)
coordinator, bus, restarted = _coordinator(tmp_path)
await coordinator.scan()
assert bus.inbound.empty()
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
result = await coordinator.handle_action(
"continue",
{"chat_id": "chat", "recovery_id": state["recovery_id"]},
)
assert result["status"] == "resuming"
assert bus.inbound.get_nowait().metadata["_webui_recovery_id"] == state["recovery_id"]
@pytest.mark.asyncio
async def test_new_user_message_supersedes_waiting_recovery(tmp_path: Path) -> None:
coordinator, bus, sessions = _coordinator(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.append({"role": "user", "content": "old request"})
session.metadata[PENDING_USER_TURN_KEY] = True
_persist(sessions, session)
await coordinator.scan()
newer = InboundMessage(
channel="websocket",
sender_id="user",
chat_id="chat",
content="new request",
)
assert await coordinator.admit(newer) is True
restored = sessions.get_or_create("websocket:chat")
assert restored.messages[-1]["_recovery_interrupted"] is True
assert sum(
message.get("_recovery_interrupted") is True
for message in restored.messages
) == 1
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "superseded"
@pytest.mark.asyncio
async def test_new_user_message_cancels_active_recovery_task(tmp_path: Path) -> None:
coordinator, bus, sessions = _coordinator(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.metadata[RECOVERY_METADATA_KEY] = {
"status": "resuming",
"recovery_id": "active",
"attempts": 1,
}
_persist(sessions, session)
started = asyncio.Event()
async def _active_recovery() -> None:
started.set()
await asyncio.Event().wait()
task = asyncio.create_task(_active_recovery())
await started.wait()
coordinator.register_recovery_task("websocket:chat", task)
newer = InboundMessage(
channel="websocket",
sender_id="user",
chat_id="chat",
content="new request",
)
assert await coordinator.admit(newer) is True
assert task.cancelled()
restored = sessions.get_or_create("websocket:chat")
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "superseded"
assert restored.messages[-1]["_recovery_interrupted"] is True
first = bus.outbound.get_nowait().event
second = bus.outbound.get_nowait().event
assert isinstance(first, RecoveryStateEvent)
assert isinstance(second, SessionUpdatedEvent)
assert bus.outbound.empty()
@pytest.mark.asyncio
async def test_recovery_action_rejects_stale_page_and_continues_current_state(
tmp_path: Path,
) -> None:
coordinator, bus, sessions = _coordinator(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.metadata[RECOVERY_METADATA_KEY] = {
"status": "awaiting_user",
"recovery_id": "current",
"attempts": 0,
}
_persist(sessions, session)
with pytest.raises(RecoveryActionError, match="stale"):
await coordinator.handle_action(
"continue",
{"chat_id": "chat", "recovery_id": "old"},
)
result = await coordinator.handle_action(
"continue",
{"chat_id": "chat", "recovery_id": "current"},
)
assert result["status"] == "resuming"
assert bus.inbound.get_nowait().metadata["_webui_recovery_id"] == "current"
@pytest.mark.asyncio
async def test_persisted_completion_wins_over_stale_resuming_marker(tmp_path: Path) -> None:
sessions = SessionManager(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.messages.extend(
[
{"role": "user", "content": "work"},
{"role": "assistant", "content": "done"},
]
)
session.metadata[RECOVERY_METADATA_KEY] = {
"status": "resuming",
"recovery_id": "recovery",
"attempts": 1,
"resume_message_count": 1,
}
_persist(sessions, session)
coordinator, bus, restarted = _coordinator(tmp_path)
await coordinator.scan()
assert bus.inbound.empty()
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
assert state["status"] == "recovered"
assert state["reason"] == "committed"
@pytest.mark.asyncio
async def test_dismiss_does_not_queue_work(tmp_path: Path) -> None:
coordinator, bus, sessions = _coordinator(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.metadata[RECOVERY_METADATA_KEY] = {
"status": "awaiting_user",
"recovery_id": "current",
"attempts": 0,
}
_persist(sessions, session)
result = await coordinator.handle_action(
"dismiss",
{"chat_id": "chat", "recovery_id": "current"},
)
assert result["status"] == "recovered"
assert bus.inbound.empty()
@pytest.mark.asyncio
async def test_scan_failure_is_visible_instead_of_aborting_other_sessions(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
coordinator, bus, sessions = _coordinator(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.metadata[PENDING_USER_TURN_KEY] = True
_persist(sessions, session)
async def fail(*_args: object, **_kwargs: object) -> None:
raise RuntimeError("boom")
monkeypatch.setattr(RecoveryCoordinator, "_recover_session", fail)
await coordinator.scan()
state = sessions.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
assert state["status"] == "failed"
assert state["can_continue"] is False
assert isinstance(bus.outbound.get_nowait().event, RecoveryStateEvent)
with pytest.raises(RecoveryActionError, match="context is unavailable"):
await coordinator.handle_action(
"continue",
{"chat_id": "chat", "recovery_id": state["recovery_id"]},
)
@pytest.mark.asyncio
async def test_bus_remains_quiet_after_recovered_state(tmp_path: Path) -> None:
coordinator, bus, sessions = _coordinator(tmp_path)
session = sessions.get_or_create("websocket:chat")
session.metadata[RECOVERY_METADATA_KEY] = {
"status": "recovered",
"recovery_id": "done",
"attempts": 1,
}
_persist(sessions, session)
await coordinator.scan()
await asyncio.sleep(0)
assert bus.inbound.empty()
assert bus.outbound.empty()
tests/session/test_session_store.py
+91
View File
@@ -1,6 +1,7 @@
from unittest.mock import MagicMock
import nanobot.session as session_api
from nanobot.providers.base import ProviderConversationState
from nanobot.session import Session, SessionManager
from nanobot.session.manager import SessionStore
from nanobot.session.model_selection import SESSION_MODEL_PRESET_METADATA_KEY
@@ -123,3 +124,93 @@ def test_manager_preserves_full_session_before_store_save(tmp_path) -> None:
assert session.messages[0]["content"] == "0"
assert session.messages[-1]["content"] == "2000"
store.save.assert_called_once_with(session, fsync=False)
def test_runtime_checkpoint_does_not_rewrite_long_session(tmp_path) -> None:
manager = SessionManager(tmp_path)
session = manager.get_or_create("websocket:long")
for index in range(256):
session.add_message("user", f"{index}:" + "x" * 4096)
manager.save(session)
main_path = manager._get_session_path(session.key)
main_before = main_path.read_bytes()
stat_before = main_path.stat()
session.metadata["runtime_checkpoint"] = {
"phase": "tools_completed",
"assistant_message": {"role": "assistant", "content": "working"},
"completed_tool_results": [],
"pending_tool_calls": [],
}
session.provider_state = ProviderConversationState(
kind="openai_responses",
provider="openai:test",
model="test-model",
version=1,
payload={"response_id": "private-response"},
)
manager.save_runtime_checkpoint(session)
checkpoint_path = manager._get_runtime_checkpoint_path(session.key)
assert main_path.read_bytes() == main_before
assert main_path.stat().st_ino == stat_before.st_ino
assert main_path.stat().st_mtime_ns == stat_before.st_mtime_ns
assert checkpoint_path.stat().st_size < len(main_before) // 100
restored = SessionManager(tmp_path).get_or_create(session.key)
assert restored.metadata["runtime_checkpoint"]["phase"] == "tools_completed"
assert restored.provider_state is not None
assert restored.provider_state.payload == {"response_id": "private-response"}
def test_completed_session_supersedes_stale_checkpoint(tmp_path) -> None:
manager = SessionManager(tmp_path)
session = manager.get_or_create("websocket:completed")
session.add_message("user", "question")
manager.save(session)
session.metadata["runtime_checkpoint"] = {"phase": "awaiting_tools"}
manager.save_runtime_checkpoint(session)
checkpoint_path = manager._get_runtime_checkpoint_path(session.key)
stale_checkpoint = checkpoint_path.read_bytes()
session.metadata.pop("runtime_checkpoint")
session.add_message("assistant", "answer")
manager.save(session)
assert not checkpoint_path.exists()
# Emulate a process dying after the main record was committed but before an
# obsolete sidecar could be removed. The base fingerprint keeps it stale.
checkpoint_path.write_bytes(stale_checkpoint)
restored = SessionManager(tmp_path).get_or_create(session.key)
assert "runtime_checkpoint" not in restored.metadata
assert restored.messages[-1]["content"] == "answer"
assert not checkpoint_path.exists()
def test_delete_session_removes_runtime_checkpoint(tmp_path) -> None:
manager = SessionManager(tmp_path)
session = manager.get_or_create("websocket:delete")
session.add_message("user", "question")
manager.save(session)
session.metadata["runtime_checkpoint"] = {"phase": "awaiting_tools"}
manager.save_runtime_checkpoint(session)
checkpoint_path = manager._get_runtime_checkpoint_path(session.key)
assert checkpoint_path.exists()
assert manager.delete_session(session.key) is True
assert not checkpoint_path.exists()
def test_invalid_runtime_checkpoint_is_discarded(tmp_path) -> None:
manager = SessionManager(tmp_path)
session = manager.get_or_create("websocket:invalid-checkpoint")
session.add_message("user", "question")
manager.save(session)
checkpoint_path = manager._get_runtime_checkpoint_path(session.key)
checkpoint_path.write_text("{truncated", encoding="utf-8")
restored = SessionManager(tmp_path).get_or_create(session.key)
assert "runtime_checkpoint" not in restored.metadata
assert not checkpoint_path.exists()
tests/utils/test_webui_transcript.py
+18
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from pathlib import Path
import nanobot.webui.transcript as transcript_module
from nanobot.session.history_visibility import HIDDEN_HISTORY_META
from nanobot.webui.transcript import (
@@ -554,6 +556,22 @@ def test_thread_response_marks_unfinished_tool_tail_pending(tmp_path, monkeypatc
assert out["completed_turn_ids"] == []
def test_recovery_tail_check_reads_only_the_active_transcript(tmp_path, monkeypatch) -> None:
monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
key = "websocket:recovery-tail"
active_path = transcript_module.webui_transcript_path(key)
reads: list[Path] = []
def read(path: Path) -> list[dict[str, object]]:
reads.append(path)
return [{"event": "message", "kind": "progress", "text": "running"}]
monkeypatch.setattr(transcript_module, "_read_transcript_file", read)
assert transcript_module.has_unfinished_transcript_tail(key) is True
assert reads == [active_path]
def test_thread_response_reports_active_registry_without_transcript(
tmp_path,
monkeypatch,
tests/webui/test_gateway_webui_smoke.py
+84
View File
@@ -15,6 +15,9 @@ import httpx
import pytest
import websockets
from nanobot.session.manager import SessionManager
from nanobot.session.recovery import PENDING_USER_TURN_KEY, RUNTIME_CHECKPOINT_KEY
_BOOTSTRAP_SECRET = "smoke-secret"
@@ -206,3 +209,84 @@ async def test_gateway_webui_bootstrap_message_and_thread_hydration(tmp_path: Pa
assert any("shell-ok" in text for text in contents)
finally:
_stop_gateway(process)
def test_gateway_restart_restores_a_completed_answer_without_replaying_model(
tmp_path: Path,
) -> None:
"""Exercise recovery through two real gateway processes and durable files."""
ws_port = _free_port()
gateway_port = _free_port()
workspace = tmp_path / "workspace"
workspace.mkdir()
config_path = tmp_path / "config.json"
first_log = tmp_path / "gateway-first.log"
second_log = tmp_path / "gateway-second.log"
_write_smoke_config(
config_path,
workspace=workspace,
ws_port=ws_port,
gateway_port=gateway_port,
)
base_url = f"http://127.0.0.1:{ws_port}"
first = _start_gateway(config_path, first_log)
try:
_wait_for_bootstrap(base_url, first, first_log)
finally:
_stop_gateway(first)
sessions_root = tmp_path / "sessions"
sessions = SessionManager(workspace, sessions_root=sessions_root)
session = sessions.get_or_create("websocket:recovery-smoke")
session.messages.append({"role": "user", "content": "recover this answer"})
session.metadata["webui"] = True
session.metadata[PENDING_USER_TURN_KEY] = True
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
"phase": "final_response",
"assistant_message": {
"role": "assistant",
"content": "restored without another model request",
},
"completed_tool_results": [],
"pending_tool_calls": [],
}
sessions.save(session, fsync=True)
second = _start_gateway(config_path, second_log)
try:
bootstrap = _wait_for_bootstrap(base_url, second, second_log)
deadline = time.monotonic() + 20
restored = None
while time.monotonic() < deadline:
restored = SessionManager(
workspace,
sessions_root=sessions_root,
).get_or_create("websocket:recovery-smoke")
if any(
message.get("content") == "restored without another model request"
for message in restored.messages
):
break
time.sleep(0.1)
else:
logs = second_log.read_text(encoding="utf-8", errors="replace")
raise AssertionError(f"answer was not recovered after restart\n{logs}")
assert restored is not None
assert PENDING_USER_TURN_KEY not in restored.metadata
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
assert restored.metadata["webui_recovery"]["reason"] == "answer_restored"
async def assert_attach_state() -> None:
ws_url = f'{bootstrap["ws_url"]}?token={bootstrap["token"]}&client_id=recovery-smoke'
async with websockets.connect(ws_url) as ws:
await _recv_until(ws, "ready")
await ws.send(json.dumps({"type": "attach", "chat_id": "recovery-smoke"}))
attached = await _recv_until(ws, "attached")
assert attached["recovery_state"]["status"] == "recovered"
assert attached["recovery_state"]["reason"] == "answer_restored"
asyncio.run(assert_attach_state())
finally:
_stop_gateway(second)
tests/webui/test_session_list_index.py
+25
View File
@@ -18,6 +18,7 @@ from nanobot.session.automation_turns import AUTOMATION_HISTORY_META
from nanobot.session.history_visibility import HIDDEN_HISTORY_META
from nanobot.session.manager import SessionManager
from nanobot.session.model_selection import SESSION_MODEL_PRESET_METADATA_KEY
from nanobot.session.recovery import RECOVERY_METADATA_KEY
@pytest.fixture(autouse=True)
@@ -66,6 +67,30 @@ def test_webui_session_list_refreshes_after_model_preset_rename(tmp_path: Path)
assert list_webui_sessions(manager)[0]["model_preset"] == "Codex"
def test_webui_session_list_surfaces_pending_recovery_state(tmp_path: Path) -> None:
manager = SessionManager(tmp_path)
session = manager.get_or_create("websocket:needs-attention")
session.add_message("user", "the interrupted task")
session.metadata[RECOVERY_METADATA_KEY] = {
"status": "awaiting_user",
"recovery_id": "recovery-123",
"reason": "uncertain_tool_state",
"attempts": 1,
# Private checkpoint details must never leak into the sidebar index.
"checkpoint": {"tool_args": "secret"},
}
manager.save(session)
row = list_webui_sessions(manager)[0]
assert row["recovery_state"] == {
"status": "awaiting_user",
"recovery_id": "recovery-123",
"reason": "uncertain_tool_state",
"attempts": 1,
}
def test_webui_session_index_uses_unique_temp_file(tmp_path: Path) -> None:
manager = SessionManager(tmp_path)
session = manager.get_or_create("websocket:unique-index-temp")
tui/src/app.test.ts
+119 -1
View File
@@ -7,7 +7,12 @@ import {
} from "@opentui/core/testing"
import { NanobotTui, sessionExitMessage, type AppOptions } from "./app"
import type { MessageOptions, SlashCommand, WorkspaceScopePayload } from "./protocol"
import type {
MessageOptions,
RecoveryState,
SlashCommand,
WorkspaceScopePayload,
} from "./protocol"
import type { HostAgentState, HostMetadata, TuiHost } from "./host"
const options: AppOptions = {
@@ -91,6 +96,13 @@ function client(
setWorkspaceScope(scope: WorkspaceScopePayload) {
scopes.push(scope)
},
updateRecovery(
_action: "continue" | "dismiss",
_chatId: string,
recoveryId: string,
): Promise<RecoveryState> {
return Promise.resolve({ status: "recovered" as const, recovery_id: recoveryId })
},
}
}
@@ -903,6 +915,112 @@ describe("NanobotTui layout", () => {
}
})
test("offers clickable recovery actions without letting a late response revive stale state", async () => {
setup = await createRenderer({ width: 96, height: 24, screenMode: "alternate-screen" })
const calls: Array<{ action: string; chatId: string; recoveryId: string }> = []
let deferredResolve: ((state: RecoveryState) => void) | undefined
const recoveryClient = client()
recoveryClient.updateRecovery = (action, chatId, recoveryId) => {
calls.push({ action, chatId, recoveryId })
if (recoveryId === "recovery-1") {
return Promise.resolve({ status: "resuming", recovery_id: recoveryId })
}
if (action === "dismiss") {
return Promise.resolve({ status: "recovered", recovery_id: recoveryId })
}
return new Promise((resolve) => { deferredResolve = resolve })
}
const app = NanobotTui.mount(
setup.renderer,
options,
recoveryClient,
new MockTreeSitterClient({ autoResolveTimeout: 0 }),
)
app.accept({
event: "attached",
chat_id: "chat",
recovery_state: {
status: "awaiting_user",
recovery_id: "recovery-1",
reason: "tool execution interrupted",
},
})
const ui = app as unknown as {
activeTurn: boolean
composer: TextareaRenderable
recoveryNotice: {
visible: boolean
dismiss: TextRenderable
resume: TextRenderable
}
status: TextRenderable
}
await waitUntil(() => (app as unknown as { ready: boolean }).ready)
await setup.renderOnce()
expect(setup.captureCharFrame()).toContain("Task interrupted")
expect(ui.activeTurn).toBe(false)
expect(ui.composer.focused).toBe(true)
await setup.mockMouse.click(ui.recoveryNotice.resume.x + 1, ui.recoveryNotice.resume.y)
await waitUntil(() => calls.length === 1 && ui.activeTurn)
expect(calls[0]).toEqual({
action: "continue",
chatId: "chat",
recoveryId: "recovery-1",
})
expect(ui.recoveryNotice.visible).toBe(false)
expect(ui.status.plainText).toContain("Continuing")
app.accept({
event: "recovery_state",
chat_id: "chat",
status: "awaiting_user",
recovery_id: "recovery-2",
})
await setup.renderOnce()
await setup.mockMouse.click(ui.recoveryNotice.resume.x + 1, ui.recoveryNotice.resume.y)
await waitUntil(() => calls.length === 2)
app.accept({
event: "recovery_state",
chat_id: "chat",
status: "recovered",
recovery_id: "recovery-2",
})
deferredResolve?.({ status: "resuming", recovery_id: "recovery-2" })
await Bun.sleep(1)
expect(ui.recoveryNotice.visible).toBe(false)
expect(ui.activeTurn).toBe(false)
expect(ui.composer.focused).toBe(true)
app.accept({
event: "recovery_state",
chat_id: "chat",
status: "awaiting_user",
recovery_id: "recovery-3",
})
await setup.renderOnce()
await setup.mockMouse.click(ui.recoveryNotice.dismiss.x + 1, ui.recoveryNotice.dismiss.y)
await waitUntil(() => calls.length === 3 && !ui.recoveryNotice.visible)
expect(calls[2]).toEqual({
action: "dismiss",
chatId: "chat",
recoveryId: "recovery-3",
})
app.accept({
event: "recovery_state",
chat_id: "chat",
status: "awaiting_user",
recovery_id: "recovery-4",
can_continue: false,
})
await setup.renderOnce()
expect(ui.recoveryNotice.resume.visible).toBe(false)
expect(ui.recoveryNotice.dismiss.visible).toBe(true)
})
test("preserves gateway slash lifecycle while local navigation stays in the same menu", async () => {
setup = await createRenderer({ width: 80, height: 24, screenMode: "alternate-screen" })
const sent: string[] = []
tui/src/app.ts
+124 -1
View File
@@ -34,6 +34,7 @@ import {
type InboundEvent,
type MentionCandidate,
type MessageOptions,
type RecoveryState,
type SlashCommand,
type SessionSummary,
type TokenUsage,
@@ -70,6 +71,7 @@ import {
} from "./mention-menu"
import { PromptQueue, type QueuedPrompt } from "./prompt-queue"
import { QueuePreview, type QueuePreviewTheme } from "./queue-preview"
import { RecoveryNotice, type RecoveryNoticeTheme } from "./recovery-notice"
import { RuntimeControls } from "./runtime-controls"
import {
contextualFooterHints,
@@ -107,6 +109,11 @@ interface ChatClient {
newChat(scope?: WorkspaceScopePayload): void
forkChat?(sourceChatId: string, beforeUserIndex: number, title?: string): void
setWorkspaceScope(scope: WorkspaceScopePayload): void
updateRecovery(
action: "continue" | "dismiss",
chatId: string,
recoveryId: string,
): Promise<RecoveryState>
}
interface Palette {
@@ -295,6 +302,15 @@ function queuePreviewTheme(palette: Palette): QueuePreviewTheme {
}
}
function recoveryNoticeTheme(palette: Palette): RecoveryNoticeTheme {
return {
text: palette.text,
muted: palette.muted,
accent: palette.accent,
error: palette.error,
}
}
function footerHintTheme(palette: Palette): FooterHintTheme {
return {
accent: palette.accent,
@@ -380,6 +396,7 @@ export class NanobotTui {
private readonly contextPanel: ContextPanel
private readonly diffViewer: DiffViewer
private readonly queuePreview: QueuePreview
private readonly recoveryNotice: RecoveryNotice
private readonly client: ChatClient
private readonly shell: BoxRenderable
private readonly title: BoxRenderable
@@ -444,6 +461,8 @@ export class NanobotTui {
private currentTask = ""
private currentAction = ""
private hostBlocked = false
private recoveryState: RecoveryState | null = null
private recoveryPending = false
private hostWorkspace: string
private hostBranch: string
private readonly apiReauthenticator: ApiReauthenticator | undefined
@@ -495,6 +514,14 @@ export class NanobotTui {
treeSitterClient,
)
this.queuePreview = new QueuePreview(renderer, queuePreviewTheme(this.palette))
this.recoveryNotice = new RecoveryNotice(
renderer,
recoveryNoticeTheme(this.palette),
{
onContinue: () => void this.updateRecovery("continue"),
onDismiss: () => void this.updateRecovery("dismiss"),
},
)
this.client = client || new NanobotClient({
...(options.bootstrapUrl
? {
@@ -723,6 +750,7 @@ export class NanobotTui {
this.shell.add(this.runtimeControls.menuRoot)
if (!host.hosted) this.shell.add(this.title)
this.shell.add(this.queuePreview.root)
this.shell.add(this.recoveryNotice.root)
this.shell.add(this.composerFrame)
this.shell.add(statusRow)
this.shell.add(this.diffViewer.root)
@@ -831,6 +859,12 @@ export class NanobotTui {
this.quit()
return
}
if (["/continue", "/dismiss"].includes(visibleContent.toLowerCase())) {
this.clearComposer()
this.commandMenu.hide()
void this.updateRecovery(visibleContent.toLowerCase() === "/continue" ? "continue" : "dismiss")
return
}
const completion = this.commandMenu.completion(visibleContent)
if (completion) {
this.setComposer(completion)
@@ -946,7 +980,9 @@ export class NanobotTui {
}
const hydrationId = ++this.hydrationId
void this.prepareChat(event.chat_id, restoring, hydrationId).then(() => {
if (hydrationId === this.hydrationId) this.flushPendingEvents()
if (hydrationId !== this.hydrationId) return
this.applyRecoveryState(event.recovery_state ?? null)
this.flushPendingEvents()
})
return
}
@@ -1083,6 +1119,9 @@ export class NanobotTui {
this.applyHostGoalState(event.goal_state)
if (!this.activeTurn) this.reportHostResting()
return
case "recovery_state":
this.applyRecoveryState(event)
return
case "turn_model_updated":
if (typeof event.context_window_tokens === "number") {
this.contextWindowTokens = event.context_window_tokens
@@ -1191,6 +1230,85 @@ export class NanobotTui {
for (const event of events || []) this.accept(event)
}
private clearRecoveryState(): void {
this.recoveryState = null
this.recoveryPending = false
this.recoveryNotice.hide()
}
private applyRecoveryState(state: RecoveryState | null): void {
if (!state) {
this.clearRecoveryState()
return
}
this.recoveryState = state
this.recoveryPending = false
if (state.status === "resuming") {
this.recoveryNotice.hide()
this.hostBlocked = false
this.activeLabel = "Continuing"
this.setCurrentAction("Continuing interrupted task")
this.setActive(true)
this.reportHostWorking()
return
}
if (state.status === "awaiting_user" || state.status === "failed") {
this.activeTurnId = null
this.setActive(false)
this.hostBlocked = true
this.recoveryNotice.show(state)
const detail = state.reason || (state.status === "failed"
? "Recovery failed"
: "Task interrupted")
this.setCurrentAction(detail)
this.status.content = "Waiting for recovery decision"
this.host.reportState("blocked", detail)
this.composer.focus()
return
}
this.clearRecoveryState()
this.activeTurnId = null
this.hostBlocked = false
this.setActive(false)
if (this.ready) this.status.content = this.readyStatus()
this.reportHostResting()
}
private async updateRecovery(action: "continue" | "dismiss"): Promise<void> {
const state = this.recoveryState
if (
!state
|| (state.status !== "awaiting_user" && state.status !== "failed")
|| (action === "continue" && state.can_continue === false)
) {
this.status.content = "No interrupted task"
this.composer.focus()
return
}
if (this.recoveryPending) return
this.recoveryPending = true
this.recoveryNotice.setBusy(true)
this.status.content = action === "continue" ? "Continuing…" : "Dismissing…"
try {
const next = await this.client.updateRecovery(
action,
this.client.activeChatId,
state.recovery_id,
)
if (this.recoveryState?.recovery_id === state.recovery_id) {
this.applyRecoveryState(next)
}
} catch (error) {
if (this.recoveryState?.recovery_id !== state.recovery_id) return
this.recoveryPending = false
this.recoveryNotice.setBusy(false)
this.status.content = error instanceof Error ? error.message : String(error)
this.host.reportState("blocked", state.reason || "Task interrupted")
} finally {
this.composer.focus()
}
}
private updateGatewayApiConnection(apiUrl: string, apiToken: string): void {
this.options.apiUrl = apiUrl
this.options.apiToken = apiToken
@@ -1576,6 +1694,7 @@ export class NanobotTui {
this.contextPanel.setTheme(contextPanelTheme(this.palette))
this.diffViewer.setTheme(diffViewerTheme(this.palette, this.backgroundKnown))
this.queuePreview.setTheme(queuePreviewTheme(this.palette))
this.recoveryNotice.setTheme(recoveryNoticeTheme(this.palette))
this.updateComposerAppearance()
this.composer.textColor = this.palette.text
this.composer.focusedTextColor = this.palette.text
@@ -2001,6 +2120,7 @@ export class NanobotTui {
this.sessionTitle = sessionLabel(current)
this.applySessionModel(current)
this.applySessionScope(current)
this.applyRecoveryState(current.recoveryState ?? null)
this.updateTitle()
}
const limit = this.renderer.height >= 20 ? 8 : 4
@@ -2027,6 +2147,7 @@ export class NanobotTui {
this.sessionTitle = sessionLabel(session)
this.applySessionModel(session)
this.applySessionScope(session)
this.applyRecoveryState(session.recoveryState ?? null)
this.updateTitle()
this.closeSessions()
this.status.content = this.readyStatus()
@@ -2039,6 +2160,7 @@ export class NanobotTui {
this.closeSessions()
try {
this.ready = false
this.clearRecoveryState()
this.clearPromptQueue()
this.sessionMetadataId += 1
this.clearHostContext()
@@ -2073,6 +2195,7 @@ export class NanobotTui {
this.clearComposer()
try {
this.ready = false
this.clearRecoveryState()
this.clearPromptQueue()
this.sessionMetadataId += 1
this.clearHostContext()
tui/src/protocol.test.ts
+56
View File
@@ -595,6 +595,62 @@ describe("gateway protocol", () => {
}
})
test("sends stale-safe recovery mutations and validates their response", async () => {
const original = globalThis.WebSocket
let socket: FakeSocket | undefined
Object.defineProperty(globalThis, "WebSocket", {
configurable: true,
value: class extends FakeSocket {
constructor() {
super()
socket = this
}
},
})
try {
const statuses: string[] = []
const client = new NanobotClient({
url: "ws://nanobot.test/ws",
onEvent: () => undefined,
onStatus: (status, detail) => statuses.push(`${status}:${detail || ""}`),
})
client.connect()
if (!socket) throw new Error("socket was not created")
const result = client.updateRecovery("continue", "chat", "recovery-1")
const request = JSON.parse(socket.sent.at(-1) || "{}") as {
request_id: string
action: string
payload: Record<string, string>
}
expect(request).toMatchObject({
type: "webui_request",
action: "recovery.continue",
payload: { chat_id: "chat", recovery_id: "recovery-1" },
})
socket.emit("message", { data: JSON.stringify({
event: "webui_response",
request_id: request.request_id,
ok: true,
result: { status: "resuming", recovery_id: "recovery-1", attempts: 1 },
}) })
expect(await result).toEqual({
status: "resuming",
recovery_id: "recovery-1",
attempts: 1,
})
expect(statuses).not.toContain("error:gateway sent an invalid event")
const interrupted = client.updateRecovery("dismiss", "chat", "recovery-2")
socket.emit("close")
await expect(interrupted).rejects.toThrow("gateway connection closed")
client.close()
} finally {
Object.defineProperty(globalThis, "WebSocket", { configurable: true, value: original })
}
})
test("reports when the bounded history snapshot omits earlier turns", async () => {
const original = globalThis.fetch
let requested = ""
tui/src/protocol.ts
+132 -1
View File
@@ -54,6 +54,16 @@ export interface RuntimeControls {
canUseFullAccess: boolean
}
export type RecoveryStatus = "resuming" | "awaiting_user" | "recovered" | "failed"
export interface RecoveryState {
status: RecoveryStatus
recovery_id: string
reason?: string
attempts?: number
can_continue?: boolean
}
export type InboundEvent =
| { event: "ready"; chat_id: string; client_id: string }
| {
@@ -61,6 +71,7 @@ export type InboundEvent =
chat_id: string
model_preset?: string | null
usage?: TokenUsage
recovery_state?: RecoveryState
}
| {
event: "message_accepted"
@@ -118,6 +129,7 @@ export type InboundEvent =
turn_id?: string
}
| { event: "goal_state"; chat_id: string; goal_state: Record<string, unknown> }
| ({ event: "recovery_state"; chat_id: string } & RecoveryState)
| {
event: "session_updated"
chat_id: string
@@ -139,6 +151,12 @@ type OutboundEvent =
| { type: "fork_chat"; source_chat_id: string; before_user_index: number; title?: string }
| { type: "attach"; chat_id: string }
| { type: "set_workspace_scope"; chat_id: string; workspace_scope: WorkspaceScopePayload }
| {
type: "webui_request"
request_id: string
action: string
payload: Record<string, unknown>
}
| {
type: "message"
chat_id: string
@@ -271,6 +289,7 @@ export interface SessionSummary {
updatedAt: string | null
runStartedAt: number | null
modelPreset: string | null
recoveryState?: RecoveryState | null
workspaceScope?: WorkspaceScopePayload | null
pinned: boolean
archived: boolean
@@ -297,6 +316,7 @@ const CHAT_EVENTS = new Set([
"turn_end",
"goal_status",
"goal_state",
"recovery_state",
"session_updated",
"turn_model_updated",
"error",
@@ -377,6 +397,34 @@ function isWorkspaceScope(value: unknown): value is WorkspaceScopePayload {
&& optional(value.restrict_to_workspace, "boolean")
}
function isRecoveryState(value: unknown): value is RecoveryState {
return isRecord(value)
&& ["resuming", "awaiting_user", "recovered", "failed"].includes(String(value.status))
&& typeof value.recovery_id === "string"
&& optional(value.reason, "string")
&& optional(value.attempts, "number")
&& optional(value.can_continue, "boolean")
}
interface WebUIResponseEvent {
event: "webui_response"
request_id: string
ok: boolean
result?: unknown
error?: { status: number; message: string }
}
function decodeWebUIResponse(value: unknown): WebUIResponseEvent | null | undefined {
if (!isRecord(value) || value.event !== "webui_response") return undefined
if (typeof value.request_id !== "string" || typeof value.ok !== "boolean") return null
if (value.ok) return value as unknown as WebUIResponseEvent
return isRecord(value.error)
&& typeof value.error.status === "number"
&& typeof value.error.message === "string"
? value as unknown as WebUIResponseEvent
: null
}
function decodeInboundEvent(value: unknown): InboundEvent | null | undefined {
if (!isRecord(value)) return null
const record = value
@@ -407,7 +455,8 @@ function decodeInboundEvent(value: unknown): InboundEvent | null | undefined {
&& ((record.model_preset !== undefined
&& record.model_preset !== null
&& typeof record.model_preset !== "string")
|| (record.usage !== undefined && !isTokenUsage(record.usage)))
|| (record.usage !== undefined && !isTokenUsage(record.usage))
|| (record.recovery_state !== undefined && !isRecoveryState(record.recovery_state)))
) return null
if (
["user_message", "message", "delta", "reasoning_delta"].includes(name)
@@ -452,6 +501,7 @@ function decodeInboundEvent(value: unknown): InboundEvent | null | undefined {
) return null
if (name === "goal_status" && record.status !== "running" && record.status !== "idle") return null
if (name === "goal_state" && !isRecord(record.goal_state)) return null
if (name === "recovery_state" && !isRecoveryState(record)) return null
if (
name === "session_updated"
&& (!optional(record.scope, "string")
@@ -700,6 +750,9 @@ export async function fetchSessions(
modelPreset: typeof value.model_preset === "string" && value.model_preset.trim()
? value.model_preset.trim()
: null,
...(isRecoveryState(value.recovery_state)
? { recoveryState: value.recovery_state }
: {}),
...(isWorkspaceScope(value.workspace_scope) ? { workspaceScope: value.workspace_scope } : {}),
pinned: pinned.has(value.key),
archived: archived.has(value.key),
@@ -855,6 +908,11 @@ export class NanobotClient {
private closedByClient = false
private opening = false
private connectedOnce = false
private readonly pendingMutations = new Map<string, {
resolve: (value: unknown) => void
reject: (error: Error) => void
timer: ReturnType<typeof setTimeout>
}>()
constructor(private readonly options: ClientOptions) {}
@@ -916,6 +974,7 @@ export class NanobotClient {
socket.addEventListener("close", () => {
if (this.socket !== socket) return
this.socket = null
this.rejectPendingMutations("gateway connection closed")
if (this.closedByClient) {
this.options.onStatus("closed")
return
@@ -931,6 +990,7 @@ export class NanobotClient {
const socket = this.socket
this.socket = null
socket?.close()
this.rejectPendingMutations("gateway connection closed")
}
send(content: string, options: MessageOptions = {}): string {
@@ -975,6 +1035,63 @@ export class NanobotClient {
this.write({ type: "set_workspace_scope", chat_id: this.chatId, workspace_scope: scope })
}
updateRecovery(
action: "continue" | "dismiss",
chatId: string,
recoveryId: string,
): Promise<RecoveryState> {
return this.requestMutation<unknown>(`recovery.${action}`, {
chat_id: chatId,
recovery_id: recoveryId,
}).then((result) => {
if (!isRecoveryState(result)) throw new Error("gateway returned an invalid recovery state")
return result
})
}
private requestMutation<T>(
action: string,
payload: Record<string, unknown> = {},
timeoutMs = 20_000,
): Promise<T> {
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
return Promise.reject(new Error("gateway connection is not open"))
}
const requestId = crypto.randomUUID()
const frame = JSON.stringify({
type: "webui_request",
request_id: requestId,
action,
payload,
} satisfies OutboundEvent)
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => {
this.pendingMutations.delete(requestId)
reject(new Error(`gateway request timed out after ${timeoutMs}ms`))
}, timeoutMs)
this.pendingMutations.set(requestId, {
resolve: (value) => resolve(value as T),
reject,
timer,
})
try {
this.socket?.send(frame)
} catch {
clearTimeout(timer)
this.pendingMutations.delete(requestId)
reject(new Error("could not send gateway request"))
}
})
}
private rejectPendingMutations(message: string): void {
for (const pending of this.pendingMutations.values()) {
clearTimeout(pending.timer)
pending.reject(new Error(message))
}
this.pendingMutations.clear()
}
private handleMessage(raw: string): void {
let value: unknown
try {
@@ -983,6 +1100,20 @@ export class NanobotClient {
this.options.onStatus("error", "gateway sent invalid JSON")
return
}
const response = decodeWebUIResponse(value)
if (response === null) {
this.options.onStatus("error", "gateway sent an invalid event")
return
}
if (response) {
const pending = this.pendingMutations.get(response.request_id)
if (!pending) return
clearTimeout(pending.timer)
this.pendingMutations.delete(response.request_id)
if (response.ok) pending.resolve(response.result)
else pending.reject(new Error(response.error?.message || "gateway request failed"))
return
}
const event = decodeInboundEvent(value)
if (event === undefined) return
if (event === null) {
tui/src/recovery-notice.ts
+155
View File
@@ -0,0 +1,155 @@
import {
BoxRenderable,
RGBA,
StyledText,
TextAttributes,
TextRenderable,
type CliRenderer,
type TextChunk,
} from "@opentui/core"
import type { RecoveryState } from "./protocol"
export interface RecoveryNoticeTheme {
text: string
muted: string
accent: string
error: string
}
interface RecoveryNoticeOptions {
onContinue: () => void
onDismiss: () => void
}
/** A quiet action surface for a gateway-owned interrupted turn. */
export class RecoveryNotice {
readonly root: BoxRenderable
private readonly message: TextRenderable
private readonly dismiss: TextRenderable
private readonly resume: TextRenderable
private state: RecoveryState | null = null
private busy = false
constructor(
renderer: CliRenderer,
private theme: RecoveryNoticeTheme,
options: RecoveryNoticeOptions,
) {
this.root = new BoxRenderable(renderer, {
id: "nanobot-tui-recovery-notice",
width: "100%",
height: 1,
flexShrink: 0,
flexDirection: "row",
alignItems: "center",
gap: 2,
paddingLeft: 1,
paddingRight: 1,
visible: false,
backgroundColor: RGBA.defaultBackground(),
})
this.message = new TextRenderable(renderer, {
id: "nanobot-tui-recovery-message",
width: "auto",
minWidth: 0,
flexGrow: 1,
height: 1,
truncate: true,
selectable: false,
})
this.dismiss = this.action(renderer, "dismiss", "Dismiss", options.onDismiss)
this.resume = this.action(renderer, "continue", "Continue", options.onContinue, true)
this.root.add(this.message)
this.root.add(this.dismiss)
this.root.add(this.resume)
}
get visible(): boolean {
return this.root.visible
}
show(state: RecoveryState): void {
this.state = state
this.busy = false
this.root.visible = true
this.render()
}
hide(): void {
this.state = null
this.busy = false
this.root.visible = false
}
setBusy(busy: boolean): void {
this.busy = busy
if (this.visible) this.render()
}
setTheme(theme: RecoveryNoticeTheme): void {
this.theme = theme
if (this.visible) this.render()
}
private action(
renderer: CliRenderer,
id: string,
label: string,
callback: () => void,
primary = false,
): TextRenderable {
return new TextRenderable(renderer, {
id: `nanobot-tui-recovery-${id}`,
content: label,
width: label.length,
height: 1,
flexShrink: 0,
selectable: false,
onMouseOver: () => {
if (this.busy) return
const target = primary ? this.resume : this.dismiss
target.attributes = TextAttributes.BOLD | TextAttributes.UNDERLINE
},
onMouseOut: () => this.render(),
onMouseDown: (event) => {
if (event.button !== 0 || this.busy) return
event.preventDefault()
event.stopPropagation()
renderer.clearSelection()
callback()
},
})
}
private render(): void {
if (!this.state) return
const failed = this.state.status === "failed"
const contextUnavailable = this.state.can_continue === false
const title = failed ? "Recovery failed" : "Task interrupted"
const detail = failed
? "Review the task before continuing"
: contextUnavailable
? "Saved context unavailable"
: "Tools will not replay automatically"
this.message.content = new StyledText([
chunk("△ ", failed ? this.theme.error : this.theme.accent),
chunk(title, this.theme.text, true),
chunk(` · ${detail}`, this.theme.muted),
])
this.dismiss.fg = RGBA.fromHex(this.busy ? this.theme.muted : this.theme.text)
this.resume.visible = !contextUnavailable
this.resume.fg = RGBA.fromHex(this.busy ? this.theme.muted : this.theme.accent)
this.dismiss.attributes = 0
this.resume.attributes = TextAttributes.BOLD
}
}
function chunk(text: string, color: string, bold = false): TextChunk {
return {
__isChunk: true,
text,
fg: RGBA.fromHex(color),
attributes: bold ? TextAttributes.BOLD : 0,
}
}
tui/src/session-menu.test.ts
+6 -1
View File
@@ -24,6 +24,11 @@ const sessions: SessionSummary[] = [
updatedAt: "2026-08-12T10:00:00Z",
runStartedAt: null,
modelPreset: null,
recoveryState: {
status: "awaiting_user",
recovery_id: "recovery-two",
reason: "tool execution interrupted",
},
pinned: false,
archived: false,
},
@@ -53,7 +58,7 @@ describe("SessionMenu", () => {
menu.update("release stable", 6)
await setup.renderOnce()
expect(setup.captureCharFrame()).toContain("Release checklist")
expect(setup.captureCharFrame()).toContain("△ Release checklist")
expect(menu.choose()?.chatId).toBe("two")
})
tui/src/session-menu.ts
+7 -1
View File
@@ -42,6 +42,8 @@ export class SessionMenu {
session.chatId,
session.workspaceScope?.project_name || "",
session.workspaceScope?.project_path || "",
session.recoveryState?.status || "",
session.recoveryState?.reason || "",
].join(" "),
render: (session) => {
const age = updatedLabel(session.updatedAt)
@@ -54,7 +56,11 @@ export class SessionMenu {
]
.filter(Boolean)
.join(" · ")
const marker = session.active ? "● " : session.pinned ? "◆ " : session.archived ? "◇ " : ""
const interrupted = session.recoveryState?.status === "awaiting_user"
|| session.recoveryState?.status === "failed"
const marker = interrupted
? "△ "
: session.active ? "● " : session.pinned ? "◆ " : session.archived ? "◇ " : ""
return `${marker}${sessionLabel(session)}${detail ? ` ${detail}` : ""}`
},
emptyText: "No matching sessions",
webui/src/App.tsx
+10
View File
@@ -1274,6 +1274,15 @@ function Shell({
}, [activeKey, activeTabKey, activeTabState]);
const runningChatIdList = useMemo(() => Array.from(runningChatIds), [runningChatIds]);
const updatedChatIdList = useMemo(() => Array.from(updatedChatIds), [updatedChatIds]);
const recoveryChatIdList = useMemo(
() => sessions
.filter((session) => (
session.recoveryState?.status === "awaiting_user"
|| session.recoveryState?.status === "failed"
))
.map((session) => session.chatId),
[sessions],
);
const activeChatId = activePaneSession?.chatId ?? null;
useEffect(() => {
activeChatIdRef.current = activeChatId;
@@ -2552,6 +2561,7 @@ function Shell({
collapsedGroups: sidebarState.collapsed_groups,
runningChatIds: runningChatIdList,
updatedChatIds: updatedChatIdList,
recoveryChatIds: recoveryChatIdList,
viewState: { ...sidebarState.view, sort: automaticSidebarSort },
showArchived: sidebarState.view.show_archived,
archivedCount: sidebarArchivedTabKeys.length,
webui/src/components/ChatList.tsx
+35 -7
View File
@@ -11,6 +11,7 @@ import type { CSSProperties, MouseEvent as ReactMouseEvent, ReactElement } from
import {
Archive,
ArchiveRestore,
AlertTriangle,
ChevronDown,
Folder,
FolderTree,
@@ -260,6 +261,7 @@ interface ChatListProps {
collapsedGroups?: Record<string, boolean>;
runningChatIds?: string[];
updatedChatIds?: string[];
recoveryChatIds?: string[];
density?: SidebarDensity;
showPreviews?: boolean;
showTimestamps?: boolean;
@@ -302,6 +304,7 @@ export const ChatList = memo(function ChatList({
collapsedGroups = {},
runningChatIds = [],
updatedChatIds = [],
recoveryChatIds = [],
density = "comfortable",
showPreviews = false,
showTimestamps = false,
@@ -558,6 +561,7 @@ export const ChatList = memo(function ChatList({
const running = new Set(runningChatIds);
const updated = new Set(updatedChatIds);
const recovery = new Set(recoveryChatIds);
const compact = density === "compact";
const firstProjectGroupIndex = limitedGroups.findIndex((group) => group.kind === "project");
const selectableDeleteKeys = Array.from(new Set(limitedGroups.flatMap((group) => (
@@ -881,6 +885,7 @@ export const ChatList = memo(function ChatList({
compact={compact}
running={running}
updated={updated}
recovery={recovery}
onSelectPane={onSelectPane}
onRequestDelete={onRequestDelete}
onRequestRename={onRequestRename}
@@ -915,9 +920,11 @@ export const ChatList = memo(function ChatList({
: "";
const activityState = running.has(s.chatId)
? "running"
: updated.has(s.chatId) && !topicActive
? "updated"
: null;
: recovery.has(s.chatId)
? "recovery"
: updated.has(s.chatId) && !topicActive
? "updated"
: null;
const hasPaneMoveTarget = Boolean(onAttachPane)
&& paneGroupTargets.some((target) => (
target.key !== paneGroup?.tabKey && !target.atCapacity
@@ -1330,6 +1337,7 @@ function ActivePaneRows({
compact,
running,
updated,
recovery,
onSelectPane,
onRequestDelete,
onRequestRename,
@@ -1354,6 +1362,7 @@ function ActivePaneRows({
compact: boolean;
running: ReadonlySet<string>;
updated: ReadonlySet<string>;
recovery: ReadonlySet<string>;
onSelectPane?: (tabKey: string, paneKey: string) => void;
onRequestDelete: (key: string, label: string) => void;
onRequestRename: (key: string, label: string) => void;
@@ -1390,9 +1399,11 @@ function ActivePaneRows({
const active = tabActive && pane.key === group.activePaneKey;
const activityState = running.has(pane.chatId)
? "running"
: updated.has(pane.chatId) && !active
? "updated"
: null;
: recovery.has(pane.chatId)
? "recovery"
: updated.has(pane.chatId) && !active
? "updated"
: null;
const paneActionsLabel = t("workbench.paneActions", { title: pane.title });
const selected = selectedDeleteKeys.has(pane.key);
const isPinned = pinned.has(pane.key);
@@ -1852,10 +1863,27 @@ function ChatsFoldFooter({
function SessionActivityIndicator({
state,
}: {
state: "running" | "updated" | null;
state: "running" | "updated" | "recovery" | null;
}) {
const { t } = useTranslation();
if (state === "recovery") {
const label = t("chat.activity.recovery", {
defaultValue: "This conversation needs your attention",
});
return (
<SidebarItemTooltip label={label}>
<span
role="img"
aria-label={label}
className="grid h-4 w-4 shrink-0 place-items-center text-[#ff8a3d]"
>
<AlertTriangle className="h-3.5 w-3.5" strokeWidth={2} aria-hidden />
</span>
</SidebarItemTooltip>
);
}
if (state === "running") {
const label = t("chat.activity.running");
return (
webui/src/components/Sidebar.tsx
+2
View File
@@ -82,6 +82,7 @@ interface SidebarProps {
collapsedGroups?: Record<string, boolean>;
runningChatIds?: string[];
updatedChatIds?: string[];
recoveryChatIds?: string[];
viewState?: SidebarViewState;
showArchived?: boolean;
archivedCount?: number;
@@ -270,6 +271,7 @@ export function Sidebar(props: SidebarProps) {
collapsedGroups={props.collapsedGroups}
runningChatIds={props.runningChatIds}
updatedChatIds={props.updatedChatIds}
recoveryChatIds={props.recoveryChatIds}
density={props.viewState?.density}
showPreviews={props.viewState?.show_previews}
showTimestamps={props.viewState?.show_timestamps}
webui/src/components/settings/overview/OverviewSettings.tsx
+31
View File
@@ -421,6 +421,37 @@ export function AppearanceSettings({
label={localPrefs.brandLogos ? tx("settings.values.on", "On") : tx("settings.values.off", "Off")}
/>
</SettingsRow>
<SettingsRow
title={tx("settings.rows.browserNotifications", "Task notifications")}
description={tx(
"settings.help.browserNotifications",
"Notify only when this page is in the background. Off by default.",
)}
>
<ToggleButton
checked={localPrefs.browserNotifications}
onChange={(enabled) => {
if (!enabled) {
onChangeLocalPrefs((prev) => ({ ...prev, browserNotifications: false }));
return;
}
if (typeof Notification === "undefined") return;
if (Notification.permission === "granted") {
onChangeLocalPrefs((prev) => ({ ...prev, browserNotifications: true }));
return;
}
void Notification.requestPermission().then((permission) => {
if (permission === "granted") {
onChangeLocalPrefs((prev) => ({ ...prev, browserNotifications: true }));
}
});
}}
ariaLabel={tx("settings.rows.browserNotifications", "Task notifications")}
label={localPrefs.browserNotifications
? tx("settings.values.on", "On")
: tx("settings.values.off", "Off")}
/>
</SettingsRow>
</SettingsGroup>
</section>
</div>
webui/src/components/thread/RecoveryNotice.tsx
+108
View File
@@ -0,0 +1,108 @@
import { useEffect, useState } from "react";
import { AlertTriangle, LoaderCircle, RotateCcw, X } from "lucide-react";
import { useTranslation } from "react-i18next";
import { Button } from "@/components/ui/button";
import { cn } from "@/lib/utils";
import type { RecoveryState } from "@/lib/types";
interface RecoveryNoticeProps {
state: RecoveryState;
onContinue: () => Promise<void>;
onDismiss: () => Promise<void>;
}
export function RecoveryNotice({ state, onContinue, onDismiss }: RecoveryNoticeProps) {
const { t } = useTranslation();
const [pending, setPending] = useState<"continue" | "dismiss" | null>(null);
const [error, setError] = useState<string | null>(null);
const [hiddenRecoveryId, setHiddenRecoveryId] = useState<string | null>(null);
useEffect(() => {
// A continuation can be interrupted again with the same recovery ID.
// Reveal the decision surface when the server returns to a waiting state.
if (state.status === "awaiting_user" || state.status === "failed") {
setHiddenRecoveryId(null);
}
}, [state.recovery_id, state.status]);
if (state.status === "recovered" || hiddenRecoveryId === state.recovery_id) return null;
const waiting = state.status === "awaiting_user" || state.status === "failed";
const contextUnavailable = state.can_continue === false;
const title = state.status === "failed"
? t("recovery.failed", { defaultValue: "Task recovery failed" })
: waiting
? t("recovery.interrupted", { defaultValue: "Task interrupted" })
: t("recovery.resuming", { defaultValue: "Restoring interrupted task…" });
const detail = state.status === "failed" || contextUnavailable
? t("recovery.failedHelp", {
defaultValue: "The saved task could not be restored safely. Review it before continuing.",
})
: waiting
? t("recovery.review", { defaultValue: "Review the task before continuing. Tools will not be replayed automatically." })
: t("recovery.safeResume", { defaultValue: "Continuing from saved conversation context." });
const run = (action: "continue" | "dismiss") => {
setPending(action);
setError(null);
// ``resuming`` is an internal transition, not another task for the user
// to monitor. Hide the notice optimistically and only bring it back if
// the explicit action is rejected.
if (action === "continue") setHiddenRecoveryId(state.recovery_id);
const operation = action === "continue" ? onContinue() : onDismiss();
void operation.catch(() => {
if (action === "continue") setHiddenRecoveryId(null);
setError(t("recovery.actionFailed", { defaultValue: "Recovery action failed. Try again." }));
}).finally(() => setPending(null));
};
return (
<div
role={waiting ? "alert" : "status"}
aria-live={waiting ? "assertive" : "polite"}
aria-busy={state.status === "resuming"}
data-recovery-status={state.status}
className="mx-auto mb-2 flex w-full max-w-[49.5rem] items-center gap-3 rounded-control border border-border/70 bg-muted/35 px-3 py-2 text-sm transition-[background-color,border-color,opacity,transform] duration-200 ease-out motion-reduce:transition-none animate-in fade-in-0 slide-in-from-bottom-1 duration-200 motion-reduce:animate-none"
>
{waiting ? (
<AlertTriangle className="h-4 w-4 shrink-0 text-amber-600 dark:text-amber-400" aria-hidden />
) : (
<LoaderCircle className="h-4 w-4 shrink-0 animate-spin text-primary motion-reduce:animate-none" aria-hidden />
)}
<div className="min-w-0 flex-1">
<p className="font-medium">
{title}
</p>
<p className={cn(
"mt-0.5 text-xs",
error ? "text-destructive" : "text-muted-foreground",
)}>
{error ?? detail}
</p>
</div>
{waiting ? (
<div className="flex shrink-0 items-center gap-1.5">
<Button
type="button"
size="sm"
variant="outline"
disabled={pending !== null}
onClick={() => run("dismiss")}
>
<X className="mr-1 h-3.5 w-3.5" aria-hidden />
{t("recovery.dismiss", { defaultValue: "Dismiss" })}
</Button>
{!contextUnavailable ? (
<Button
type="button"
size="sm"
disabled={pending !== null}
onClick={() => run("continue")}
>
<RotateCcw className="mr-1 h-3.5 w-3.5" aria-hidden />
{t("recovery.continue", { defaultValue: "Continue" })}
</Button>
) : null}
</div>
) : null}
</div>
);
}
webui/src/components/thread/ThreadShell.tsx
+20 -2
View File
@@ -7,6 +7,7 @@ import { FilePreviewAvailabilityProvider } from "@/components/FilePreviewAvailab
import { FilePreviewPanel } from "@/components/FilePreviewPanel";
import { SessionHandleLabel } from "@/components/SessionHandleLabel";
import { PromptNavigator } from "@/components/thread/PromptNavigator";
import { RecoveryNotice } from "@/components/thread/RecoveryNotice";
import { SessionInfoPopover } from "@/components/thread/SessionInfoPopover";
import {
ThreadComposer,
@@ -763,6 +764,9 @@ export function ThreadShell({
isStreaming,
runStartedAt,
goalState,
recoveryState,
continueRecovery,
dismissRecovery,
send,
transcribeAudio,
stop,
@@ -835,8 +839,15 @@ export function ThreadShell({
[displayMessages],
);
const currentGoalState = messagesReady ? goalState : undefined;
const currentRunStartedAt = messagesReady ? runStartedAt : null;
const turnActive = messagesReady && (isStreaming || currentRunStartedAt !== null);
// Decision states freeze the interrupted turn and hand the next action to
// the recovery notice. ``resuming`` remains active; ``recovered`` is only
// historical metadata and must not suppress a later normal turn.
const recoveryNeedsDecision = recoveryState?.status === "awaiting_user"
|| recoveryState?.status === "failed";
const currentRunStartedAt = messagesReady && !recoveryNeedsDecision ? runStartedAt : null;
const turnActive = messagesReady
&& !recoveryNeedsDecision
&& (isStreaming || currentRunStartedAt !== null);
const restoredViewportTurnId = useMemo(
() => turnActive ? latestActiveTurnId(displayMessages, currentRunStartedAt) : null,
[currentRunStartedAt, displayMessages, turnActive],
@@ -1472,6 +1483,13 @@ export function ThreadShell({
const composer = (
<>
{recoveryState ? (
<RecoveryNotice
state={recoveryState}
onContinue={continueRecovery}
onDismiss={dismissRecovery}
/>
) : null}
{streamError && !hasInlineDeliveryError(messages, streamError) ? (
<StreamErrorNotice
error={streamError}
webui/src/hooks/useNanobotStream.ts
+102
View File
@@ -1,4 +1,5 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { useTranslation } from "react-i18next";
import { useClient } from "@/providers/ClientProvider";
import { toMediaAttachment } from "@/lib/media";
@@ -28,6 +29,7 @@ import {
} from "@/lib/thread-event-projection";
import type { UIMessageTurnFields } from "@/lib/thread-event-projection";
import { formatQuotedUserMessage } from "@/lib/user-message-quote";
import { readLocalPreferences } from "@/lib/local-preferences";
import type {
InboundEvent,
OutboundCliAppMention,
@@ -36,6 +38,7 @@ import type {
SessionMention,
GoalStateWsPayload,
MessageDeliveryStatus,
RecoveryState,
UIMediaAttachment,
UIMessage,
WorkspaceScopePayload,
@@ -244,6 +247,9 @@ export function useNanobotStream(
runStartedAt: number | null;
/** Latest sustained goal for this ``chatId`` (``goal_state`` WS events). */
goalState: GoalStateWsPayload | undefined;
recoveryState: RecoveryState | null;
continueRecovery: () => Promise<void>;
dismissRecovery: () => Promise<void>;
send: (
content: string,
images?: SendAttachment[],
@@ -262,6 +268,7 @@ export function useNanobotStream(
dismissStreamError: () => void;
} {
const { client } = useClient();
const { t } = useTranslation();
const initialRunStartedAt = chatId ? client.getRunStartedAt(chatId) : null;
const [messages, setMessages] = useState<UIMessage[]>(initialMessages);
const [messageOwnerChatId, setMessageOwnerChatId] = useState(chatId);
@@ -273,6 +280,7 @@ export function useNanobotStream(
/** Unix epoch seconds when the current user turn started; cleared on ``idle``. */
const [runStartedAt, setRunStartedAt] = useState<number | null>(initialRunStartedAt);
const [goalState, setGoalState] = useState<GoalStateWsPayload | undefined>(undefined);
const [recoveryState, setRecoveryState] = useState<RecoveryState | null>(null);
const [streamError, setStreamError] = useState<StreamError | null>(null);
const buffer = useRef<StreamBuffer | null>(null);
const activeAssistantRef = useRef<ActiveAssistantCursor | null>(null);
@@ -288,6 +296,16 @@ export function useNanobotStream(
const dismissStreamError = useCallback(() => setStreamError(null), []);
const notifyInBackground = useCallback((body: string) => {
if (
typeof Notification === "undefined"
|| Notification.permission !== "granted"
|| document.visibilityState === "visible"
|| !readLocalPreferences().browserNotifications
) return;
new Notification("nanobot", { body });
}, []);
const clearPendingStreamWork = useCallback(() => {
if (streamFrameRef.current !== null) {
window.cancelAnimationFrame(streamFrameRef.current);
@@ -639,6 +657,7 @@ export function useNanobotStream(
setStreamError(null);
setRunStartedAt(restoredRunStartedAt);
setGoalState(chatId ? client.getGoalState(chatId) : undefined);
setRecoveryState(null);
buffer.current = null;
activeAssistantRef.current = null;
closedAssistantStreamIdsRef.current.clear();
@@ -846,10 +865,71 @@ export function useNanobotStream(
return finalized;
});
suppressStreamUntilTurnEndRef.current = false;
notifyInBackground(t("recovery.completed", { defaultValue: "Task completed" }));
onTurnEnd?.();
return;
}
if (ev.event === "recovery_state") {
const next: RecoveryState = {
status: ev.status,
recovery_id: ev.recovery_id,
...(ev.reason ? { reason: ev.reason } : {}),
...(typeof ev.attempts === "number" ? { attempts: ev.attempts } : {}),
...(typeof ev.can_continue === "boolean"
? { can_continue: ev.can_continue }
: {}),
};
setRecoveryState(next);
if (ev.status === "resuming") {
setRunStartedAt((current) => current ?? Date.now() / 1000);
setIsStreaming(true);
}
if (
ev.status === "awaiting_user"
|| ev.status === "recovered"
|| ev.status === "failed"
) {
// Recovery is an explicit boundary. The interrupted turn is no
// longer running, so do not let the stale start time keep the
// activity clock (or composer stop state) alive underneath the
// recovery notice.
setRunStartedAt(null);
setIsStreaming(false);
client.finishRunLocally(chatId);
clearPendingStreamWork();
closeActiveAssistantStream();
clearActivitySegment();
if (ev.status !== "recovered") {
notifyInBackground(
ev.status === "failed"
? t("recovery.failed", { defaultValue: "Task recovery failed" })
: t("recovery.interrupted", { defaultValue: "Task interrupted" }),
);
}
}
return;
}
if (ev.event === "attached") {
setRecoveryState(ev.recovery_state ?? null);
if (ev.recovery_state?.status === "resuming") {
setRunStartedAt((current) => current ?? Date.now() / 1000);
setIsStreaming(true);
} else if (
ev.recovery_state?.status === "awaiting_user"
|| ev.recovery_state?.status === "failed"
) {
setRunStartedAt(null);
setIsStreaming(false);
client.finishRunLocally(chatId);
clearPendingStreamWork();
closeActiveAssistantStream();
clearActivitySegment();
}
return;
}
if (ev.event === "message") {
if (
suppressStreamUntilTurnEndRef.current &&
@@ -1062,8 +1142,10 @@ export function useNanobotStream(
ensureActivitySegmentId,
flushPendingStreamEvents,
isSideChannelEvent,
notifyInBackground,
onTurnEnd,
schedulePendingStreamFlush,
t,
]);
const send = useCallback(
@@ -1173,12 +1255,32 @@ export function useNanobotStream(
[client],
);
const recoveryAction = useCallback(async (action: "continue" | "dismiss") => {
if (!chatId || !recoveryState) return;
await client.requestMutation(`recovery.${action}`, {
chat_id: chatId,
recovery_id: recoveryState.recovery_id,
});
}, [chatId, client, recoveryState]);
const continueRecovery = useCallback(
() => recoveryAction("continue"),
[recoveryAction],
);
const dismissRecovery = useCallback(
() => recoveryAction("dismiss"),
[recoveryAction],
);
return {
messages,
messagesReady: messageOwnerChatId === chatId,
isStreaming,
runStartedAt,
goalState,
recoveryState,
continueRecovery,
dismissRecovery,
send,
transcribeAudio,
stop,
webui/src/i18n/locales/en/common.json
+16 -1
View File
@@ -198,6 +198,7 @@
"fileEditDisplay": "File edit display",
"codeWrap": "Code wrapping",
"brandLogos": "Brand logos",
"browserNotifications": "Task notifications",
"maxResults": "Max results",
"timeout": "Timeout",
"jinaReader": "Jina reader",
@@ -243,6 +244,7 @@
"fileEditDisplay": "Choose whether file edit activity opens as line counts or a diff.",
"codeWrap": "Keep long code lines readable on smaller screens.",
"brandLogos": "Show third-party provider and CLI logos in Settings.",
"browserNotifications": "Notify only when this page is in the background. Off by default.",
"maxResults": "Results returned by each web_search call.",
"timeout": "Seconds before a search provider request times out.",
"jinaReader": "Use Jina Reader for web_fetch when available.",
@@ -1013,7 +1015,8 @@
"activity": {
"running": "Agent running",
"complete": "Agent finished",
"updated": "New activity"
"updated": "New activity",
"recovery": "This conversation needs your attention"
},
"pin": "Pin",
"unpin": "Unpin",
@@ -1436,6 +1439,18 @@
"estimated": "Includes estimated usage"
}
},
"recovery": {
"actionFailed": "Recovery action failed. Try again.",
"interrupted": "Task interrupted",
"completed": "Task completed",
"failed": "Task recovery failed",
"failedHelp": "The saved task could not be restored safely. Review it before continuing.",
"resuming": "Restoring interrupted task…",
"review": "Review the task before continuing. Tools will not be replayed automatically.",
"safeResume": "Continuing from saved conversation context.",
"dismiss": "Dismiss",
"continue": "Continue"
},
"lightbox": {
"title": "Image preview",
"open": "View image",
webui/src/i18n/locales/es/common.json
+16 -1
View File
@@ -161,6 +161,7 @@
"webuiDefaultAccess": "Acceso predeterminado",
"currentModel": "Configuración actual",
"brandLogos": "Logos de marca",
"browserNotifications": "Notificaciones de tareas",
"cliAppsCatalog": "Catálogo",
"cliAppsFilter": "Filtro",
"engine": "Motor",
@@ -204,6 +205,7 @@
"selectedModelProvider": "Definido por el modelo seleccionado.",
"selectedModelValue": "Definido por el modelo seleccionado.",
"brandLogos": "Muestra logos de proveedores de terceros y CLI en Ajustes.",
"browserNotifications": "Notifica solo cuando esta página está en segundo plano. Desactivado de forma predeterminada.",
"cliAppsCatalog": "Instala solo adaptadores CLI de aplicaciones que nanobot puede ejecutar localmente; las aplicaciones nativas no se modifican.",
"cliAppsFilter": "Busca por aplicación, categoría o capacidad.",
"logs": "Abre la carpeta de registros del motor nativo.",
@@ -1000,7 +1002,8 @@
"activity": {
"running": "Agente en ejecución",
"complete": "Agente terminado",
"updated": "Nueva actividad"
"updated": "Nueva actividad",
"recovery": "Esta conversación requiere tu atención"
},
"pin": "Fijar",
"unpin": "Desfijar",
@@ -1423,6 +1426,18 @@
"automationSourceFallback": "Automatización",
"automationTriggered": "Activada automáticamente"
},
"recovery": {
"actionFailed": "La recuperación falló. Inténtalo de nuevo.",
"interrupted": "Tarea interrumpida",
"completed": "Tarea completada",
"failed": "La recuperación de la tarea falló",
"failedHelp": "La tarea guardada no se pudo restaurar de forma segura. Revísala antes de continuar.",
"resuming": "Restaurando la tarea interrumpida…",
"review": "Revisa la tarea antes de continuar. Las herramientas no se repetirán automáticamente.",
"safeResume": "Continuando desde el contexto guardado de la conversación.",
"dismiss": "Descartar",
"continue": "Continuar"
},
"lightbox": {
"title": "Vista previa de imagen",
"open": "Ver imagen",
webui/src/i18n/locales/fr/common.json
+16 -1
View File
@@ -161,6 +161,7 @@
"webuiDefaultAccess": "Accès par défaut",
"currentModel": "Configuration actuelle",
"brandLogos": "Logos de marque",
"browserNotifications": "Notifications de tâches",
"cliAppsCatalog": "Catalogue",
"cliAppsFilter": "Filtre",
"engine": "Moteur",
@@ -204,6 +205,7 @@
"selectedModelProvider": "Défini par le modèle sélectionné.",
"selectedModelValue": "Défini par le modèle sélectionné.",
"brandLogos": "Affiche les logos de fournisseurs tiers et CLI dans les Réglages.",
"browserNotifications": "Notifier uniquement lorsque cette page est en arrière-plan. Désactivé par défaut.",
"cliAppsCatalog": "Installe uniquement les adaptateurs CLI d’applications que nanobot peut exécuter localement ; les applications natives restent inchangées.",
"cliAppsFilter": "Recherchez par application, catégorie ou capacité.",
"logs": "Ouvre le dossier des journaux du moteur natif.",
@@ -999,7 +1001,8 @@
"activity": {
"running": "Agent en cours",
"complete": "Agent terminé",
"updated": "Nouvelle activité"
"updated": "Nouvelle activité",
"recovery": "Cette conversation nécessite votre attention"
},
"pin": "Épingler",
"unpin": "Désépingler",
@@ -1422,6 +1425,18 @@
"automationSourceFallback": "Automatisation",
"automationTriggered": "Déclenché automatiquement"
},
"recovery": {
"actionFailed": "La récupération a échoué. Réessayez.",
"interrupted": "Tâche interrompue",
"completed": "Tâche terminée",
"failed": "Échec de la récupération de la tâche",
"failedHelp": "La tâche enregistrée n’a pas pu être restaurée en toute sécurité. Vérifiez-la avant de continuer.",
"resuming": "Restauration de la tâche interrompue…",
"review": "Vérifiez la tâche avant de continuer. Les outils ne seront pas relancés automatiquement.",
"safeResume": "Reprise depuis le contexte de conversation enregistré.",
"dismiss": "Ignorer",
"continue": "Continuer"
},
"lightbox": {
"title": "Aperçu de l’image",
"open": "Voir l’image",
webui/src/i18n/locales/id/common.json
+16 -1
View File
@@ -161,6 +161,7 @@
"webuiDefaultAccess": "Akses bawaan",
"currentModel": "Konfigurasi saat ini",
"brandLogos": "Logo merek",
"browserNotifications": "Notifikasi tugas",
"cliAppsCatalog": "Katalog",
"cliAppsFilter": "Saring",
"engine": "Mesin",
@@ -204,6 +205,7 @@
"selectedModelProvider": "Ditentukan oleh model yang dipilih.",
"selectedModelValue": "Ditentukan oleh model yang dipilih.",
"brandLogos": "Tampilkan logo penyedia pihak ketiga dan CLI di Pengaturan.",
"browserNotifications": "Beri tahu hanya saat halaman ini di latar belakang. Nonaktif secara bawaan.",
"cliAppsCatalog": "Instal hanya adaptor CLI aplikasi yang dapat dijalankan nanobot secara lokal; aplikasi asli tidak diubah.",
"cliAppsFilter": "Cari berdasarkan aplikasi, kategori, atau kemampuan.",
"logs": "Buka folder log mesin asli.",
@@ -999,7 +1001,8 @@
"activity": {
"running": "Agen sedang berjalan",
"complete": "Agen selesai",
"updated": "Aktivitas baru"
"updated": "Aktivitas baru",
"recovery": "Percakapan ini memerlukan perhatian Anda"
},
"pin": "Sematkan",
"unpin": "Lepas sematan",
@@ -1422,6 +1425,18 @@
"automationSourceFallback": "Otomatisasi",
"automationTriggered": "Dipicu otomatis"
},
"recovery": {
"actionFailed": "Pemulihan gagal. Coba lagi.",
"interrupted": "Tugas terputus",
"completed": "Tugas selesai",
"failed": "Pemulihan tugas gagal",
"failedHelp": "Tugas tersimpan tidak dapat dipulihkan dengan aman. Tinjau sebelum melanjutkan.",
"resuming": "Memulihkan tugas yang terputus…",
"review": "Tinjau tugas sebelum melanjutkan. Alat tidak akan dijalankan ulang secara otomatis.",
"safeResume": "Melanjutkan dari konteks percakapan yang tersimpan.",
"dismiss": "Abaikan",
"continue": "Lanjutkan"
},
"lightbox": {
"title": "Pratinjau gambar",
"open": "Lihat gambar",
webui/src/i18n/locales/ja/common.json
+16 -1
View File
@@ -161,6 +161,7 @@
"webuiDefaultAccess": "既定の権限",
"currentModel": "現在の設定",
"brandLogos": "ブランドロゴ",
"browserNotifications": "タスク通知",
"cliAppsCatalog": "カタログ",
"cliAppsFilter": "フィルター",
"engine": "エンジン",
@@ -204,6 +205,7 @@
"selectedModelProvider": "選択したモデルによって設定されます。",
"selectedModelValue": "選択したモデルによって設定されます。",
"brandLogos": "設定で第三者プロバイダーと CLI のロゴを表示します。",
"browserNotifications": "このページがバックグラウンドにある場合のみ通知します。既定ではオフです。",
"cliAppsCatalog": "nanobot がローカルで実行できるアプリ CLI アダプターだけをインストールします。ネイティブアプリは変更しません。",
"cliAppsFilter": "アプリ、カテゴリ、機能で検索します。",
"logs": "ネイティブエンジンのログフォルダーを開きます。",
@@ -999,7 +1001,8 @@
"activity": {
"running": "エージェント実行中",
"complete": "エージェント完了",
"updated": "新しいアクティビティ"
"updated": "新しいアクティビティ",
"recovery": "この会話には対応が必要です"
},
"pin": "ピン留め",
"unpin": "ピン留めを解除",
@@ -1422,6 +1425,18 @@
"automationSourceFallback": "自動化",
"automationTriggered": "自動実行"
},
"recovery": {
"actionFailed": "復元操作に失敗しました。もう一度お試しください。",
"interrupted": "タスクが中断されました",
"completed": "タスクが完了しました",
"failed": "タスクの復元に失敗しました",
"failedHelp": "保存されたタスクを安全に復元できませんでした。続行前に確認してください。",
"resuming": "中断されたタスクを復元しています…",
"review": "続行する前にタスクを確認してください。ツールは自動的に再実行されません。",
"safeResume": "保存された会話コンテキストから続行しています。",
"dismiss": "閉じる",
"continue": "続行"
},
"lightbox": {
"title": "画像プレビュー",
"open": "画像を表示",
webui/src/i18n/locales/ko/common.json
+16 -1
View File
@@ -161,6 +161,7 @@
"webuiDefaultAccess": "기본 권한",
"currentModel": "현재 구성",
"brandLogos": "브랜드 로고",
"browserNotifications": "작업 알림",
"cliAppsCatalog": "카탈로그",
"cliAppsFilter": "필터",
"engine": "엔진",
@@ -204,6 +205,7 @@
"selectedModelProvider": "선택한 모델에 의해 설정됩니다.",
"selectedModelValue": "선택한 모델에 의해 설정됩니다.",
"brandLogos": "설정에서 타사 제공자와 CLI 로고를 표시합니다.",
"browserNotifications": "이 페이지가 백그라운드에 있을 때만 알립니다. 기본값은 꺼짐입니다.",
"cliAppsCatalog": "nanobot이 로컬에서 실행할 수 있는 앱 CLI 어댑터만 설치합니다. 네이티브 앱은 변경하지 않습니다.",
"cliAppsFilter": "앱, 카테고리 또는 기능으로 검색합니다.",
"logs": "네이티브 엔진 로그 폴더를 엽니다.",
@@ -999,7 +1001,8 @@
"activity": {
"running": "에이전트 실행 중",
"complete": "에이전트 완료",
"updated": "새 활동"
"updated": "새 활동",
"recovery": "이 대화에는 확인이 필요합니다"
},
"pin": "고정",
"unpin": "고정 해제",
@@ -1422,6 +1425,18 @@
"automationSourceFallback": "자동화",
"automationTriggered": "자동 실행됨"
},
"recovery": {
"actionFailed": "복구 작업에 실패했습니다. 다시 시도하세요.",
"interrupted": "작업이 중단됨",
"completed": "작업 완료",
"failed": "작업 복구 실패",
"failedHelp": "저장된 작업을 안전하게 복구할 수 없습니다. 계속하기 전에 검토하세요.",
"resuming": "중단된 작업을 복구하는 중…",
"review": "계속하기 전에 작업을 검토하세요. 도구는 자동으로 다시 실행되지 않습니다.",
"safeResume": "저장된 대화 컨텍스트에서 계속합니다.",
"dismiss": "닫기",
"continue": "계속"
},
"lightbox": {
"title": "이미지 미리보기",
"open": "이미지 보기",
webui/src/i18n/locales/pt-BR/common.json
+16 -1
View File
@@ -198,6 +198,7 @@
"fileEditDisplay": "Exibição de edição de arquivo",
"codeWrap": "Quebra de linha no código",
"brandLogos": "Logos de marca",
"browserNotifications": "Notificações de tarefas",
"maxResults": "Máx. de resultados",
"timeout": "Tempo limite",
"jinaReader": "Leitor Jina",
@@ -243,6 +244,7 @@
"fileEditDisplay": "Escolha se a atividade de edição de arquivo é exibida como contagem de linhas ou como diferenças.",
"codeWrap": "Mantém linhas longas de código legíveis em telas menores.",
"brandLogos": "Mostra logotipos de provedores terceiros e de CLIs em Configurações.",
"browserNotifications": "Notifica somente quando esta página está em segundo plano. Desativado por padrão.",
"maxResults": "Resultados retornados por cada chamada de web_search.",
"timeout": "Segundos antes de uma requisição de busca expirar.",
"jinaReader": "Usa o Jina Reader para web_fetch quando disponível.",
@@ -1013,7 +1015,8 @@
"activity": {
"running": "Agente em execução",
"complete": "Agente finalizado",
"updated": "Nova atividade"
"updated": "Nova atividade",
"recovery": "Esta conversa precisa da sua atenção"
},
"pin": "Fixar",
"unpin": "Desafixar",
@@ -1436,6 +1439,18 @@
"estimated": "Inclui uso estimado"
}
},
"recovery": {
"actionFailed": "A recuperação falhou. Tente novamente.",
"interrupted": "Tarefa interrompida",
"completed": "Tarefa concluída",
"failed": "Falha ao recuperar a tarefa",
"failedHelp": "Não foi possível restaurar a tarefa salva com segurança. Revise-a antes de continuar.",
"resuming": "Restaurando a tarefa interrompida…",
"review": "Revise a tarefa antes de continuar. As ferramentas não serão executadas novamente de forma automática.",
"safeResume": "Continuando a partir do contexto de conversa salvo.",
"dismiss": "Descartar",
"continue": "Continuar"
},
"lightbox": {
"title": "Pré-visualização de imagem",
"open": "Ver imagem",
webui/src/i18n/locales/vi/common.json
+16 -1
View File
@@ -161,6 +161,7 @@
"webuiDefaultAccess": "Quyền mặc định",
"currentModel": "Cấu hình hiện tại",
"brandLogos": "Logo thương hiệu",
"browserNotifications": "Thông báo tác vụ",
"cliAppsCatalog": "Danh mục",
"cliAppsFilter": "Bộ lọc",
"engine": "Bộ máy",
@@ -204,6 +205,7 @@
"selectedModelProvider": "Được đặt bởi mô hình đã chọn.",
"selectedModelValue": "Được đặt bởi mô hình đã chọn.",
"brandLogos": "Hiển thị logo nhà cung cấp bên thứ ba và CLI trong Cài đặt.",
"browserNotifications": "Chỉ thông báo khi trang này ở nền. Mặc định tắt.",
"cliAppsCatalog": "Chỉ cài đặt các bộ chuyển đổi CLI ứng dụng mà nanobot có thể chạy cục bộ; ứng dụng gốc không bị thay đổi.",
"cliAppsFilter": "Tìm theo ứng dụng, danh mục hoặc khả năng.",
"logs": "Mở thư mục nhật ký của bộ máy gốc.",
@@ -999,7 +1001,8 @@
"activity": {
"running": "Tác nhân đang chạy",
"complete": "Tác nhân đã hoàn tất",
"updated": "Hoạt động mới"
"updated": "Hoạt động mới",
"recovery": "Cuộc trò chuyện này cần bạn xử lý"
},
"pin": "Ghim",
"unpin": "Bỏ ghim",
@@ -1422,6 +1425,18 @@
"automationSourceFallback": "Tự động hóa",
"automationTriggered": "Tự động kích hoạt"
},
"recovery": {
"actionFailed": "Khôi phục thất bại. Hãy thử lại.",
"interrupted": "Tác vụ bị gián đoạn",
"completed": "Tác vụ đã hoàn tất",
"failed": "Khôi phục tác vụ thất bại",
"failedHelp": "Không thể khôi phục an toàn tác vụ đã lưu. Hãy xem lại trước khi tiếp tục.",
"resuming": "Đang khôi phục tác vụ bị gián đoạn…",
"review": "Hãy xem lại tác vụ trước khi tiếp tục. Công cụ sẽ không tự động chạy lại.",
"safeResume": "Đang tiếp tục từ ngữ cảnh hội thoại đã lưu.",
"dismiss": "Bỏ qua",
"continue": "Tiếp tục"
},
"lightbox": {
"title": "Xem trước ảnh",
"open": "Xem ảnh",
webui/src/i18n/locales/zh-CN/common.json
+16 -1
View File
@@ -198,6 +198,7 @@
"fileEditDisplay": "文件编辑显示",
"codeWrap": "代码换行",
"brandLogos": "品牌 Logo",
"browserNotifications": "任务通知",
"maxResults": "最大结果数",
"timeout": "超时",
"jinaReader": "Jina 阅读器",
@@ -243,6 +244,7 @@
"fileEditDisplay": "选择文件编辑活动默认显示行数还是差异。",
"codeWrap": "让长代码行在小屏幕上也易读。",
"brandLogos": "在设置中显示第三方提供商和 CLI 图标。",
"browserNotifications": "仅在页面位于后台时通知，默认关闭。",
"maxResults": "每次 web_search 调用返回的结果数。",
"timeout": "搜索提供商请求超时前等待的秒数。",
"jinaReader": "可用时为 web_fetch 使用 Jina Reader。",
@@ -1013,7 +1015,8 @@
"activity": {
"running": "智能体正在运行",
"complete": "智能体已完成",
"updated": "有新内容"
"updated": "有新内容",
"recovery": "此对话需要你的处理"
},
"pin": "置顶",
"unpin": "取消置顶",
@@ -1436,6 +1439,18 @@
"estimated": "包含估算用量"
}
},
"recovery": {
"actionFailed": "恢复操作失败，请重试。",
"interrupted": "任务已中断",
"completed": "任务已完成",
"failed": "任务恢复失败",
"failedHelp": "无法安全恢复已保存的任务，继续前请先检查。",
"resuming": "正在恢复中断的任务…",
"review": "继续前请检查任务。工具不会被自动重放。",
"safeResume": "正在从已保存的对话上下文继续。",
"dismiss": "忽略",
"continue": "继续"
},
"lightbox": {
"title": "图片预览",
"open": "查看图片",
webui/src/i18n/locales/zh-TW/common.json
+16 -1
View File
@@ -161,6 +161,7 @@
"webuiDefaultAccess": "預設存取權",
"currentModel": "目前設定",
"brandLogos": "品牌 Logo",
"browserNotifications": "任務通知",
"cliAppsCatalog": "目錄",
"cliAppsFilter": "篩選",
"engine": "引擎",
@@ -204,6 +205,7 @@
"selectedModelProvider": "由選取的模型決定。",
"selectedModelValue": "由選取的模型決定。",
"brandLogos": "在設定中顯示第三方供應商與 CLI 圖示。",
"browserNotifications": "僅在頁面位於背景時通知，預設關閉。",
"cliAppsCatalog": "只安裝 nanobot 可在本機執行的應用程式專用 CLI 轉接器；不會改動原生應用程式。",
"cliAppsFilter": "依應用程式、類別或功能搜尋。",
"logs": "開啟原生引擎日誌資料夾。",
@@ -999,7 +1001,8 @@
"activity": {
"running": "智能體正在執行",
"complete": "智能體已完成",
"updated": "有新內容"
"updated": "有新內容",
"recovery": "此對話需要你的處理"
},
"pin": "置頂",
"unpin": "取消置頂",
@@ -1422,6 +1425,18 @@
"automationTriggered": "已自動觸發",
"askAboutSelection": "繼續提問"
},
"recovery": {
"actionFailed": "復原操作失敗，請再試一次。",
"interrupted": "任務已中斷",
"completed": "任務已完成",
"failed": "任務復原失敗",
"failedHelp": "無法安全復原已儲存的任務，繼續前請先檢查。",
"resuming": "正在復原中斷的任務…",
"review": "繼續前請檢查任務。工具不會自動重播。",
"safeResume": "正在從已儲存的對話上下文繼續。",
"dismiss": "略過",
"continue": "繼續"
},
"lightbox": {
"title": "圖片預覽",
"open": "檢視圖片",
webui/src/lib/api.ts
+3
View File
@@ -22,6 +22,7 @@ import type {
ProviderOAuthCompletionResult,
ProviderOAuthLoginResult,
ProviderSettingsUpdate,
RecoveryState,
SessionDeleteResult,
SessionHandle,
SessionAutomationsPayload,
@@ -192,6 +193,7 @@ export async function listSessions(
preview?: string;
model_preset?: string | null;
run_started_at?: number | null;
recovery_state?: RecoveryState | null;
workspace_scope?: WorkspaceScopePayload | null;
handle?: SessionHandle | null;
};
@@ -212,6 +214,7 @@ export async function listSessions(
preview: s.preview ?? "",
modelPreset: s.model_preset ?? null,
runStartedAt: s.run_started_at ?? null,
recoveryState: s.recovery_state ?? null,
workspaceScope: s.workspace_scope ?? null,
handle,
};
webui/src/lib/local-preferences.ts
+3
View File
@@ -7,6 +7,7 @@ export interface LocalPreferences {
activityMode: LocalActivityMode;
codeWrap: boolean;
brandLogos: boolean;
browserNotifications: boolean;
fileEditDisplayMode: FileEditDisplayMode;
}
@@ -18,6 +19,7 @@ export const DEFAULT_LOCAL_PREFS: LocalPreferences = {
activityMode: "auto",
codeWrap: true,
brandLogos: false,
browserNotifications: false,
fileEditDisplayMode: "summary",
};
@@ -35,6 +37,7 @@ export function readLocalPreferences(): LocalPreferences {
activityMode: parsed.activityMode === "expanded" ? "expanded" : "auto",
codeWrap: parsed.codeWrap !== false,
brandLogos: parsed.brandLogos === true,
browserNotifications: parsed.browserNotifications === true,
fileEditDisplayMode: normalizeFileEditDisplayMode(parsed.fileEditDisplayMode),
};
} catch {
webui/src/lib/types.ts
+17
View File
@@ -49,6 +49,16 @@ export interface TurnUsage {
[key: string]: number | undefined;
}
export type RecoveryStatus = "resuming" | "awaiting_user" | "recovered" | "failed";
export interface RecoveryState {
status: RecoveryStatus;
recovery_id: string;
reason?: string;
attempts?: number;
can_continue?: boolean;
}
export interface UIMessage {
id: string;
role: Role;
@@ -368,6 +378,8 @@ export interface ChatSummary {
modelPreset?: string | null;
/** Unix epoch seconds when this session currently has a turn in flight. */
runStartedAt?: number | null;
/** Durable recovery state that needs attention after an interrupted turn. */
recoveryState?: RecoveryState | null;
workspaceScope?: WorkspaceScopePayload | null;
/** Stable, server-owned @handle for this session. */
handle?: SessionHandle | null;
@@ -1246,6 +1258,7 @@ export type InboundEvent =
event: "attached";
chat_id: string;
temporary?: boolean;
recovery_state?: RecoveryState;
usage?: TurnUsage;
}
| {
@@ -1289,6 +1302,10 @@ export type InboundEvent =
/** Optional structured payload on progress frames (channel-specific). */
agent_ui?: AgentUIBlob;
} & InboundTurnMetadata)
| ({
event: "recovery_state";
chat_id: string;
} & RecoveryState)
| ({
event: "file_edit";
chat_id: string;
webui/src/tests/chat-list.test.tsx
+38
View File
@@ -103,6 +103,24 @@ describe("ChatList", () => {
);
});
it("marks a conversation that needs recovery attention with a warning indicator", () => {
render(
<ChatList
sessions={[session({ chatId: "recovery", title: "Interrupted task" })]}
recoveryChatIds={["recovery"]}
activeKey="websocket:other"
onSelect={vi.fn()}
onRequestDelete={vi.fn()}
onTogglePin={vi.fn()}
onRequestRename={vi.fn()}
onToggleArchive={vi.fn()}
/>,
);
expect(screen.getByRole("img", { name: "This conversation needs your attention" }))
.toBeInTheDocument();
});
it("keeps handle columns intact inside grouped panes", () => {
render(
<ChatList
@@ -151,6 +169,26 @@ describe("ChatList", () => {
}
});
it("shows the running indicator while a recovery continuation is active", () => {
render(
<ChatList
sessions={[session({ chatId: "recovery", title: "Interrupted task" })]}
runningChatIds={["recovery"]}
recoveryChatIds={["recovery"]}
activeKey="websocket:other"
onSelect={vi.fn()}
onRequestDelete={vi.fn()}
onTogglePin={vi.fn()}
onRequestRename={vi.fn()}
onToggleArchive={vi.fn()}
/>,
);
expect(screen.getByRole("img", { name: "Agent running" })).toBeInTheDocument();
expect(screen.queryByRole("img", { name: "This conversation needs your attention" }))
.not.toBeInTheDocument();
});
it("keeps tab grouping out of drag protocols while exposing inactive panes as mention sources", () => {
render(
<ChatList
webui/src/tests/i18n.test.tsx
+13
View File
@@ -161,6 +161,7 @@ const LOCALIZED_SETTINGS_COPY_KEYS = [
"settings.rows.fileEditDisplay",
"settings.rows.codeWrap",
"settings.rows.brandLogos",
"settings.rows.browserNotifications",
"settings.rows.currentModel",
"settings.rows.localServiceAccess",
"settings.rows.webuiDefaultAccess",
@@ -172,6 +173,7 @@ const LOCALIZED_SETTINGS_COPY_KEYS = [
"settings.help.fileEditDisplay",
"settings.help.codeWrap",
"settings.help.brandLogos",
"settings.help.browserNotifications",
"settings.help.currentModel",
"settings.help.localServiceAccess",
"settings.help.webuiDefaultAccess",
@@ -252,6 +254,7 @@ const LOCALIZED_NEW_SURFACE_KEYS = [
"chat.activity.running",
"chat.activity.complete",
"chat.activity.updated",
"chat.activity.recovery",
"chat.pin",
"chat.unpin",
"chat.rename",
@@ -293,6 +296,16 @@ const LOCALIZED_NEW_SURFACE_KEYS = [
"message.skill",
"settings.channels.connectionChecks",
"settings.channels.open",
"recovery.actionFailed",
"recovery.interrupted",
"recovery.completed",
"recovery.failed",
"recovery.failedHelp",
"recovery.resuming",
"recovery.review",
"recovery.safeResume",
"recovery.dismiss",
"recovery.continue",
];
const ACCIDENTALLY_SPANISH_SETTINGS_KEYS = [
"settings.help.provider",
webui/src/tests/local-preferences.test.ts
+19
View File
@@ -0,0 +1,19 @@
import { beforeEach, describe, expect, it } from "vitest";
import {
DEFAULT_LOCAL_PREFS,
readLocalPreferences,
writeLocalPreferences,
} from "@/lib/local-preferences";
describe("local preferences", () => {
beforeEach(() => localStorage.clear());
it("keeps browser notifications opt-in", () => {
expect(DEFAULT_LOCAL_PREFS.browserNotifications).toBe(false);
expect(readLocalPreferences().browserNotifications).toBe(false);
writeLocalPreferences({ ...DEFAULT_LOCAL_PREFS, browserNotifications: true });
expect(readLocalPreferences().browserNotifications).toBe(true);
});
});
webui/src/tests/recovery-notice.test.tsx
+114
View File
@@ -0,0 +1,114 @@
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
import { RecoveryNotice } from "@/components/thread/RecoveryNotice";
const INTERRUPTED = {
status: "awaiting_user" as const,
recovery_id: "recovery-1",
reason: "tool_state_uncertain",
};
describe("RecoveryNotice", () => {
it("hides the internal resuming state after Continue is accepted", async () => {
const onContinue = vi.fn().mockResolvedValue(undefined);
render(
<RecoveryNotice
state={INTERRUPTED}
onContinue={onContinue}
onDismiss={vi.fn().mockResolvedValue(undefined)}
/>,
);
fireEvent.click(screen.getByRole("button", { name: "Continue" }));
await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument());
expect(onContinue).toHaveBeenCalledOnce();
});
it("uses the shared status surface and motion treatment", () => {
render(
<RecoveryNotice
state={{ status: "resuming", recovery_id: "recovery-1" }}
onContinue={vi.fn().mockResolvedValue(undefined)}
onDismiss={vi.fn().mockResolvedValue(undefined)}
/>,
);
const notice = screen.getByRole("status");
expect(notice).toHaveAttribute("data-recovery-status", "resuming");
expect(notice).toHaveAttribute("aria-live", "polite");
expect(notice).toHaveClass(
"max-w-[49.5rem]",
"rounded-control",
"animate-in",
"fade-in-0",
"slide-in-from-bottom-1",
"duration-200",
"motion-reduce:animate-none",
);
});
it("keeps the notice visible when Continue fails", async () => {
const onContinue = vi.fn().mockRejectedValue(new Error("offline"));
render(
<RecoveryNotice
state={INTERRUPTED}
onContinue={onContinue}
onDismiss={vi.fn().mockResolvedValue(undefined)}
/>,
);
fireEvent.click(screen.getByRole("button", { name: "Continue" }));
await waitFor(() => {
expect(screen.getByRole("alert")).toHaveTextContent("Recovery action failed");
});
});
it("shows the decision surface again when a continuation is interrupted", async () => {
const onContinue = vi.fn().mockResolvedValue(undefined);
const { rerender } = render(
<RecoveryNotice
state={INTERRUPTED}
onContinue={onContinue}
onDismiss={vi.fn().mockResolvedValue(undefined)}
/>,
);
fireEvent.click(screen.getByRole("button", { name: "Continue" }));
await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument());
rerender(
<RecoveryNotice
state={{ status: "resuming", recovery_id: "recovery-1" }}
onContinue={onContinue}
onDismiss={vi.fn().mockResolvedValue(undefined)}
/>,
);
rerender(
<RecoveryNotice
state={{ ...INTERRUPTED, reason: "loop_guard" }}
onContinue={onContinue}
onDismiss={vi.fn().mockResolvedValue(undefined)}
/>,
);
await waitFor(() => expect(screen.getByRole("alert")).toBeInTheDocument());
});
it("does not offer Continue when saved conversation context is unavailable", () => {
render(
<RecoveryNotice
state={{ ...INTERRUPTED, can_continue: false }}
onContinue={vi.fn().mockResolvedValue(undefined)}
onDismiss={vi.fn().mockResolvedValue(undefined)}
/>,
);
expect(screen.queryByRole("button", { name: "Continue" })).not.toBeInTheDocument();
expect(screen.getByRole("button", { name: "Dismiss" })).toBeInTheDocument();
});
});
webui/src/tests/thread-shell.test.tsx
+28
View File
@@ -4007,4 +4007,32 @@ describe("ThreadShell", () => {
expect(screen.getByRole("option", { name: /Other project/i })).toBeInTheDocument();
});
it("allows a new turn after a completed recovery state", async () => {
const client = makeClient();
render(wrap(
client,
<ThreadShell
session={session("recovered-chat")}
title="Recovered chat"
onToggleSidebar={() => {}}
/>,
));
const input = await screen.findByLabelText("Message input");
act(() => {
client._emitChat("recovered-chat", {
event: "recovery_state",
chat_id: "recovered-chat",
recovery_id: "recovery-1",
status: "recovered",
});
});
fireEvent.change(input, { target: { value: "start the next task" } });
fireEvent.click(screen.getByRole("button", { name: "Send message" }));
expect(client.sendMessage).toHaveBeenCalledOnce();
expect(screen.getByRole("button", { name: "Stop response" })).toBeInTheDocument();
});
});
webui/src/tests/useNanobotStream.test.tsx
+98
View File
@@ -73,6 +73,7 @@ function fakeClient() {
const runStartedAtByChatId = new Map<string, number>();
const unsettledRunByChatId = new Map<string, boolean>();
const goalStateByChatId = new Map<string, GoalStateWsPayload>();
const requestMutation = vi.fn().mockResolvedValue({});
let status: ConnectionStatus = "open";
function recordGoalStatusForRunStrip(chatId: string, ev: InboundEvent) {
@@ -133,6 +134,7 @@ function fakeClient() {
return () => set!.delete(h);
},
sendMessage: vi.fn(),
requestMutation,
finishRunLocally: vi.fn(),
newChat: vi.fn(),
forkChat: vi.fn(),
@@ -157,6 +159,7 @@ function fakeClient() {
setUnsettled(chatId: string, unsettled: boolean) {
unsettledRunByChatId.set(chatId, unsettled);
},
requestMutation,
};
}
@@ -395,6 +398,101 @@ describe("useNanobotStream", () => {
});
});
it("exposes typed recovery state and validates actions with its recovery id", async () => {
const fake = fakeClient();
const { result } = renderHook(() => useNanobotStream("chat-recovery", EMPTY_MESSAGES), {
wrapper: wrap(fake.client),
});
act(() => {
fake.emit("chat-recovery", {
event: "goal_status",
chat_id: "chat-recovery",
status: "running",
started_at: 1_700,
});
});
expect(result.current.runStartedAt).toBe(1_700);
act(() => {
fake.emit("chat-recovery", {
event: "recovery_state",
chat_id: "chat-recovery",
recovery_id: "recovery-1",
status: "awaiting_user",
reason: "tool_state_uncertain",
can_continue: false,
});
});
expect(result.current.recoveryState).toEqual({
recovery_id: "recovery-1",
status: "awaiting_user",
reason: "tool_state_uncertain",
can_continue: false,
});
expect(result.current.isStreaming).toBe(false);
expect(result.current.runStartedAt).toBeNull();
expect(fake.client.finishRunLocally).toHaveBeenCalledWith("chat-recovery");
act(() => {
fake.emit("chat-recovery", {
event: "recovery_state",
chat_id: "chat-recovery",
recovery_id: "recovery-1",
status: "awaiting_user",
reason: "tool_state_uncertain",
can_continue: true,
});
});
await act(async () => result.current.continueRecovery());
expect(fake.requestMutation).toHaveBeenCalledWith("recovery.continue", {
chat_id: "chat-recovery",
recovery_id: "recovery-1",
});
act(() => {
fake.emit("chat-recovery", {
event: "recovery_state",
chat_id: "chat-recovery",
recovery_id: "recovery-1",
status: "recovered",
});
});
expect(result.current.isStreaming).toBe(false);
expect(fake.client.finishRunLocally).toHaveBeenCalledWith("chat-recovery");
});
it("does not let historical recovered state clear a later active turn", () => {
const fake = fakeClient();
const { result } = renderHook(
() => useNanobotStream("chat-recovered-history", EMPTY_MESSAGES),
{ wrapper: wrap(fake.client) },
);
act(() => {
fake.emit("chat-recovered-history", {
event: "goal_status",
chat_id: "chat-recovered-history",
status: "running",
started_at: 1_700,
});
fake.emit("chat-recovered-history", {
event: "attached",
chat_id: "chat-recovered-history",
recovery_state: {
recovery_id: "old-recovery",
status: "recovered",
},
});
});
expect(result.current.isStreaming).toBe(true);
expect(result.current.runStartedAt).toBe(1_700);
expect(fake.client.finishRunLocally).not.toHaveBeenCalled();
});
it("preserves proactive automation source metadata on complete assistant messages", () => {
const fake = fakeClient();
const { result } = renderHook(() => useNanobotStream("chat-cron", EMPTY_MESSAGES), {