mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-09-04 02:01:48 +03:00
feat(runtime): add user-controlled turn recovery
This commit is contained in:
@@ -36,6 +36,7 @@ from nanobot.session.keys import (
|
||||
UNIFIED_SESSION_KEY,
|
||||
)
|
||||
from nanobot.session.manager import Session
|
||||
from nanobot.session.recovery import PENDING_FOLLOWUP_ID_KEY, PENDING_FOLLOWUPS_KEY
|
||||
from nanobot.session.turn_continuation import (
|
||||
INTERNAL_CONTINUATION_META,
|
||||
INTERNAL_CONTINUATION_RUN_STARTED_AT_META,
|
||||
@@ -161,6 +162,35 @@ def test_persist_cron_turn_uses_distinct_history_marker(tmp_path: Path) -> None:
|
||||
assert message["cron_prompt_ref"] == prompt_ref
|
||||
|
||||
|
||||
def test_persist_user_message_acknowledges_durable_followup(tmp_path: Path) -> None:
|
||||
loop = _make_full_loop(tmp_path)
|
||||
session = loop.sessions.get_or_create("websocket:chat")
|
||||
session.metadata[PENDING_FOLLOWUPS_KEY] = [
|
||||
{
|
||||
"id": "followup-1",
|
||||
"sender_id": "user",
|
||||
"chat_id": "chat",
|
||||
"content": "queued while busy",
|
||||
"media": [],
|
||||
"metadata": {},
|
||||
}
|
||||
]
|
||||
|
||||
persisted = loop._persist_user_message_early(
|
||||
InboundMessage(
|
||||
channel="websocket",
|
||||
sender_id="user",
|
||||
chat_id="chat",
|
||||
content="queued while busy",
|
||||
metadata={PENDING_FOLLOWUP_ID_KEY: "followup-1"},
|
||||
),
|
||||
session,
|
||||
)
|
||||
|
||||
assert persisted is True
|
||||
assert PENDING_FOLLOWUPS_KEY not in session.metadata
|
||||
|
||||
|
||||
def test_persist_local_trigger_turn_uses_hidden_automation_marker(tmp_path: Path) -> None:
|
||||
loop = _make_full_loop(tmp_path)
|
||||
session = loop.sessions.get_or_create("websocket:auto")
|
||||
@@ -381,6 +411,34 @@ def test_save_turn_keeps_multimodal_runtime_context_for_model_replay() -> None:
|
||||
assert public_history_message(session.messages[0])["content"] == []
|
||||
|
||||
|
||||
def test_save_turn_acknowledges_every_merged_recovery_followup() -> None:
|
||||
"""Persisting a merged injected row retires every durable follow-up ID."""
|
||||
loop = _mk_loop()
|
||||
session = Session(
|
||||
key="test:recovery-followups",
|
||||
metadata={
|
||||
PENDING_FOLLOWUPS_KEY: [
|
||||
{"id": "first"},
|
||||
{"id": "second"},
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
loop._save_turn(
|
||||
session,
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "first\n\nsecond",
|
||||
PENDING_FOLLOWUP_ID_KEY: ["first", "second"],
|
||||
}
|
||||
],
|
||||
skip=0,
|
||||
)
|
||||
|
||||
assert PENDING_FOLLOWUPS_KEY not in session.metadata
|
||||
|
||||
|
||||
def test_save_turn_keeps_image_placeholder_and_runtime_context() -> None:
|
||||
loop = _mk_loop()
|
||||
session = Session(key="test:image")
|
||||
|
||||
@@ -26,7 +26,7 @@ def _make_injection_callback(queue: asyncio.Queue):
|
||||
return inject_cb
|
||||
|
||||
|
||||
def _make_loop(tmp_path):
|
||||
def _make_loop(tmp_path, *, recovery_admission=None):
|
||||
from nanobot.agent.loop import AgentLoop
|
||||
from nanobot.bus.queue import MessageBus
|
||||
|
||||
@@ -39,7 +39,12 @@ def _make_loop(tmp_path):
|
||||
patch("nanobot.agent.loop.SubagentManager") as mock_sub_mgr:
|
||||
mock_sub_mgr.return_value.cancel_by_session = AsyncMock(return_value=0)
|
||||
mock_sub_mgr.return_value.close = AsyncMock()
|
||||
loop = AgentLoop(bus=bus, provider=provider, workspace=tmp_path)
|
||||
loop = AgentLoop(
|
||||
bus=bus,
|
||||
provider=provider,
|
||||
workspace=tmp_path,
|
||||
recovery_admission=recovery_admission,
|
||||
)
|
||||
return loop
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -759,6 +764,20 @@ async def test_runner_merges_multiple_injected_user_messages_without_losing_medi
|
||||
)
|
||||
|
||||
|
||||
def test_runner_merge_keeps_all_recovery_followup_ids() -> None:
|
||||
"""Merged follow-ups stay acknowledged together after a later save."""
|
||||
from nanobot.agent.runner import AgentRunner
|
||||
from nanobot.session.recovery import PENDING_FOLLOWUP_ID_KEY
|
||||
|
||||
messages = [{"role": "user", "content": "first", PENDING_FOLLOWUP_ID_KEY: "one"}]
|
||||
AgentRunner._append_injected_messages(
|
||||
messages,
|
||||
[{"role": "user", "content": "second", PENDING_FOLLOWUP_ID_KEY: "two"}],
|
||||
)
|
||||
|
||||
assert messages[-1][PENDING_FOLLOWUP_ID_KEY] == ["one", "two"]
|
||||
|
||||
|
||||
def test_runner_merge_preserves_runtime_markers_with_media() -> None:
|
||||
from nanobot.agent.runner import AgentRunner
|
||||
from nanobot.runtime_context import (
|
||||
@@ -967,6 +986,38 @@ async def test_followup_routed_to_pending_queue(tmp_path):
|
||||
assert queued_msg.session_key == UNIFIED_SESSION_KEY
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_followup_is_admitted_before_recovery_queue(tmp_path):
|
||||
"""Recovery admission runs before a newer WebUI message is injected."""
|
||||
from nanobot.bus.events import InboundMessage
|
||||
|
||||
admission = MagicMock()
|
||||
admission.admit = AsyncMock(return_value=True)
|
||||
loop = _make_loop(tmp_path, recovery_admission=admission)
|
||||
loop._dispatch = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
session_key = "websocket:chat"
|
||||
pending = asyncio.Queue(maxsize=20)
|
||||
loop._pending_queues[session_key] = pending
|
||||
|
||||
run_task = asyncio.create_task(loop.run())
|
||||
msg = InboundMessage(
|
||||
channel="websocket",
|
||||
sender_id="u",
|
||||
chat_id="chat",
|
||||
content="new request",
|
||||
)
|
||||
await loop.bus.publish_inbound(msg)
|
||||
|
||||
queued_msg = await asyncio.wait_for(pending.get(), timeout=2)
|
||||
admission.admit.assert_awaited_once_with(msg)
|
||||
assert queued_msg.content == msg.content
|
||||
assert queued_msg.metadata["_recovery_followup_id"]
|
||||
|
||||
loop.stop()
|
||||
await asyncio.wait_for(run_task, timeout=2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mid_turn_subagent_result_does_not_resolve_a_new_turn_route(tmp_path):
|
||||
"""Injected results stay inside the active turn instead of opening a side turn."""
|
||||
@@ -1314,6 +1365,51 @@ async def test_pending_queue_full_falls_back_to_queued_task(tmp_path):
|
||||
assert pending.qsize() == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_queue_overflow_keeps_websocket_followup_durable(tmp_path):
|
||||
"""Fallback dispatch must not acknowledge a WebUI message before it commits."""
|
||||
from nanobot.bus.events import InboundMessage
|
||||
from nanobot.session.manager import Session
|
||||
from nanobot.session.recovery import pending_followups
|
||||
|
||||
loop = _make_loop(tmp_path)
|
||||
dispatched = asyncio.Event()
|
||||
release_dispatch = asyncio.Event()
|
||||
|
||||
async def _dispatch(_msg):
|
||||
dispatched.set()
|
||||
await release_dispatch.wait()
|
||||
|
||||
loop._dispatch = AsyncMock(side_effect=_dispatch) # type: ignore[method-assign]
|
||||
session = Session(key="websocket:c")
|
||||
loop.sessions.get_or_create.return_value = session
|
||||
pending = asyncio.Queue(maxsize=1)
|
||||
pending.put_nowait(
|
||||
InboundMessage(channel="websocket", sender_id="u", chat_id="c", content="already queued")
|
||||
)
|
||||
loop._pending_queues["websocket:c"] = pending
|
||||
|
||||
run_task = asyncio.create_task(loop.run())
|
||||
await loop.bus.publish_inbound(
|
||||
InboundMessage(
|
||||
channel="websocket",
|
||||
sender_id="u",
|
||||
chat_id="c",
|
||||
content="durable follow-up",
|
||||
metadata={"webui": True},
|
||||
)
|
||||
)
|
||||
await asyncio.wait_for(dispatched.wait(), timeout=2)
|
||||
|
||||
assert [message.content for message in pending_followups(session)] == ["durable follow-up"]
|
||||
dispatched_msg = loop._dispatch.await_args.args[0]
|
||||
assert dispatched_msg.metadata["_recovery_followup_id"]
|
||||
|
||||
release_dispatch.set()
|
||||
loop.stop()
|
||||
await asyncio.wait_for(run_task, timeout=2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_republishes_leftover_queue_messages(tmp_path):
|
||||
"""Messages left in the pending queue after _dispatch are re-published to the bus.
|
||||
|
||||
@@ -161,3 +161,25 @@ async def test_dispatch_cancellation_restores_checkpoint():
|
||||
"Checkpoint metadata should be cleared after restore"
|
||||
assert loop.sessions.save.called, \
|
||||
"Session should be persisted so the restored state survives process restart"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_cancellation_keeps_checkpoint_for_managed_restart(tmp_path: Path) -> None:
|
||||
"""A restart preserves the checkpoint; an explicit stop still restores it."""
|
||||
loop = _make_loop(tmp_path)
|
||||
loop.preserve_inflight_turns_on_shutdown()
|
||||
loop._restore_runtime_checkpoint = MagicMock() # type: ignore[method-assign]
|
||||
|
||||
async def _cancel(*_args: object, **_kwargs: object) -> None:
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
loop._process_message = _cancel # type: ignore[method-assign]
|
||||
|
||||
from nanobot.bus.events import InboundMessage
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await loop._dispatch(
|
||||
InboundMessage(channel="test", sender_id="u1", chat_id="c1", content="work")
|
||||
)
|
||||
|
||||
loop._restore_runtime_checkpoint.assert_not_called()
|
||||
|
||||
@@ -3721,6 +3721,27 @@ async def test_notify_restart_done_waits_until_channel_starts():
|
||||
assert sent_msg.content.startswith("Restart completed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_restart_notice_does_not_overwrite_recovery_state():
|
||||
"""WebSocket attach/recovery events already own reconnect state."""
|
||||
fake_config = SimpleNamespace(
|
||||
channels=ChannelsConfig(),
|
||||
providers=SimpleNamespace(groq=SimpleNamespace(api_key="")),
|
||||
)
|
||||
mgr = ChannelManager.__new__(ChannelManager)
|
||||
mgr.config = fake_config
|
||||
mgr.bus = MessageBus()
|
||||
channel = _StartableChannel(fake_config, mgr.bus)
|
||||
channel._running = True
|
||||
mgr.channels = {"websocket": channel}
|
||||
mgr._send_with_retry = AsyncMock()
|
||||
|
||||
notice = RestartNotice(channel="websocket", chat_id="chat", started_at_raw="100.0")
|
||||
await mgr._send_restart_notice_when_started(notice)
|
||||
|
||||
mgr._send_with_retry.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restart_notice_retries_until_running_channel_accepts_delivery():
|
||||
"""A running flag must not make an early transport failure final."""
|
||||
|
||||
@@ -103,6 +103,16 @@ class _GatewayAgentContractStub:
|
||||
return None
|
||||
|
||||
|
||||
class _EmptyGatewaySessionManager:
|
||||
"""Minimal session-manager contract for gateway assembly tests."""
|
||||
|
||||
def list_sessions(self) -> list[dict[str, object]]:
|
||||
return []
|
||||
|
||||
def flush_all(self) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def test_gateway_signal_handler_first_signal_stops_and_second_forces() -> None:
|
||||
class _FakeLoop:
|
||||
def __init__(self) -> None:
|
||||
@@ -2756,7 +2766,7 @@ def _patch_serve_runtime(monkeypatch, config: Config, seen: dict[str, object]) -
|
||||
monkeypatch,
|
||||
config,
|
||||
message_bus=lambda: object(),
|
||||
session_manager=lambda _workspace: object(),
|
||||
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
|
||||
)
|
||||
monkeypatch.setattr("nanobot.cli.commands.AgentLoop", _FakeAgentLoop)
|
||||
monkeypatch.setattr("nanobot.api.server.create_app", _fake_create_app)
|
||||
@@ -2823,7 +2833,7 @@ def test_gateway_uses_workspace_directory_for_cron_store(monkeypatch, tmp_path:
|
||||
monkeypatch,
|
||||
config,
|
||||
message_bus=lambda: object(),
|
||||
session_manager=lambda _workspace: object(),
|
||||
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
|
||||
cron_service=_StopCron,
|
||||
)
|
||||
|
||||
@@ -3329,7 +3339,7 @@ def test_gateway_workspace_override_does_not_migrate_legacy_cron(
|
||||
monkeypatch,
|
||||
config,
|
||||
message_bus=lambda: object(),
|
||||
session_manager=lambda _workspace: object(),
|
||||
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
|
||||
cron_service=_StopCron,
|
||||
get_cron_dir=lambda: legacy_dir,
|
||||
)
|
||||
@@ -3368,7 +3378,7 @@ def test_gateway_custom_config_workspace_does_not_migrate_legacy_cron(
|
||||
monkeypatch,
|
||||
config,
|
||||
message_bus=lambda: object(),
|
||||
session_manager=lambda _workspace: object(),
|
||||
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
|
||||
cron_service=_StopCron,
|
||||
get_cron_dir=lambda: legacy_dir,
|
||||
)
|
||||
@@ -3569,7 +3579,7 @@ def test_gateway_health_endpoint_binds_and_serves_expected_responses(
|
||||
monkeypatch,
|
||||
config,
|
||||
message_bus=lambda: object(),
|
||||
session_manager=lambda _workspace: object(),
|
||||
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
|
||||
)
|
||||
monkeypatch.setattr("nanobot.cli.gateway_runtime.AgentLoop", _FakeAgentLoop)
|
||||
monkeypatch.setattr("nanobot.channels.manager.ChannelManager", _FakeChannelManager)
|
||||
@@ -3771,7 +3781,7 @@ def test_gateway_agent_task_owns_initial_mcp_provider_close(
|
||||
monkeypatch,
|
||||
config,
|
||||
message_bus=lambda: object(),
|
||||
session_manager=lambda _workspace: object(),
|
||||
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
|
||||
)
|
||||
monkeypatch.setattr("nanobot.cli.gateway_runtime.AgentLoop", _FakeAgentLoop)
|
||||
monkeypatch.setattr("nanobot.cli.gateway_runtime.MCPProvider", _FakeMCPProvider)
|
||||
@@ -3894,7 +3904,7 @@ def test_gateway_shutdown_event_exits_forever_runtime_tasks(
|
||||
monkeypatch,
|
||||
config,
|
||||
message_bus=lambda: object(),
|
||||
session_manager=lambda _workspace: object(),
|
||||
session_manager=lambda _workspace: _EmptyGatewaySessionManager(),
|
||||
)
|
||||
monkeypatch.setattr("nanobot.cli.gateway_runtime.AgentLoop", _FakeAgentLoop)
|
||||
monkeypatch.setattr("nanobot.channels.manager.ChannelManager", _FakeChannelManager)
|
||||
|
||||
@@ -18,6 +18,7 @@ from nanobot.gateway import (
|
||||
GatewayRuntimePaths,
|
||||
GatewayStartOptions,
|
||||
GatewayStatus,
|
||||
RuntimeResult,
|
||||
)
|
||||
from nanobot.gateway.runtime import monitor_gateway_clients
|
||||
from nanobot.process_runtime import process_is_running
|
||||
@@ -454,6 +455,48 @@ def test_restart_does_not_detach_a_foreground_gateway(tmp_path, monkeypatch):
|
||||
assert result.message == "gateway_foreground_restart_required"
|
||||
|
||||
|
||||
def test_restart_marks_the_exiting_gateway_for_turn_recovery(tmp_path, monkeypatch):
|
||||
"""A managed restart must not look like an explicit stop to the child."""
|
||||
runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux")
|
||||
status = GatewayStatus(
|
||||
running=True,
|
||||
pid=12345,
|
||||
state_path=runtime.paths.state_path,
|
||||
log_path=runtime.paths.log_path,
|
||||
launch_mode="background",
|
||||
)
|
||||
monkeypatch.setattr(runtime, "status", lambda **_kwargs: status)
|
||||
|
||||
def stop(*, timeout_s: int):
|
||||
assert timeout_s == 20
|
||||
assert json.loads(runtime._restart_intent_path.read_text(encoding="utf-8")) == {
|
||||
"pid": 12345
|
||||
}
|
||||
return SimpleNamespace(ok=True, message="gateway_stopped", status=status)
|
||||
|
||||
monkeypatch.setattr(runtime, "_stop", stop)
|
||||
monkeypatch.setattr(
|
||||
runtime,
|
||||
"_start_background",
|
||||
lambda _options: RuntimeResult(True, "gateway_started_background", status),
|
||||
)
|
||||
|
||||
result = runtime.restart(GatewayStartOptions(port=18790))
|
||||
|
||||
assert result.ok is True
|
||||
assert not runtime._restart_intent_path.exists()
|
||||
|
||||
|
||||
def test_restart_intent_only_applies_to_the_recorded_gateway_process(tmp_path):
|
||||
runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux")
|
||||
|
||||
runtime._write_restart_intent(os.getpid())
|
||||
|
||||
assert runtime.preserves_inflight_turns_on_exit() is True
|
||||
runtime._write_restart_intent(os.getpid() + 1)
|
||||
assert runtime.preserves_inflight_turns_on_exit() is False
|
||||
|
||||
|
||||
def test_last_interactive_client_stops_an_on_demand_gateway(tmp_path, monkeypatch):
|
||||
runtime = GatewayRuntime(paths=_paths(tmp_path), platform_name="Linux")
|
||||
monkeypatch.setattr(runtime, "_process_identity", lambda pid: pid)
|
||||
|
||||
@@ -0,0 +1,732 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from nanobot.bus.events import InboundMessage
|
||||
from nanobot.bus.outbound_events import RecoveryStateEvent, SessionUpdatedEvent
|
||||
from nanobot.bus.queue import MessageBus
|
||||
from nanobot.session.manager import Session, SessionManager
|
||||
from nanobot.session.recovery import (
|
||||
PENDING_FOLLOWUPS_KEY,
|
||||
PENDING_USER_TURN_KEY,
|
||||
RECOVERY_METADATA_KEY,
|
||||
RUNTIME_CHECKPOINT_KEY,
|
||||
RecoveryActionError,
|
||||
RecoveryCoordinator,
|
||||
acknowledge_pending_followups,
|
||||
pending_followups,
|
||||
record_pending_followup,
|
||||
)
|
||||
from nanobot.webui import session_list_index, transcript
|
||||
|
||||
|
||||
def _persist(manager: SessionManager, session: Session) -> None:
|
||||
session.metadata["webui"] = True
|
||||
manager.save(session)
|
||||
|
||||
|
||||
def _coordinator(workspace: Path) -> tuple[RecoveryCoordinator, MessageBus, SessionManager]:
|
||||
bus = MessageBus()
|
||||
sessions = SessionManager(workspace)
|
||||
return RecoveryCoordinator(sessions, bus), bus, sessions
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restart_before_model_call_waits_for_confirmation(tmp_path: Path) -> None:
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "finish this"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, bus, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
assert bus.inbound.empty()
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
assert restored.metadata[PENDING_USER_TURN_KEY] is True
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "restart_requires_confirmation"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_incomplete_transcript_waits_for_confirmation(tmp_path: Path, monkeypatch) -> None:
|
||||
"""A materialized shutdown must not reappear as an endless Working state."""
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
_persist(sessions, session)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.webui.transcript.has_unfinished_transcript_tail",
|
||||
lambda _key: True,
|
||||
)
|
||||
|
||||
coordinator, bus, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
assert bus.inbound.empty()
|
||||
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
|
||||
assert state["status"] == "awaiting_user"
|
||||
assert state["reason"] == "interrupted_without_checkpoint"
|
||||
event = bus.outbound.get_nowait().event
|
||||
assert isinstance(event, RecoveryStateEvent)
|
||||
assert event.status == "awaiting_user"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_only_interruption_is_discovered_without_materializing_completed_history(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
webui_dir = tmp_path / "webui"
|
||||
webui_dir.mkdir()
|
||||
monkeypatch.setattr(session_list_index, "get_webui_dir", lambda: webui_dir)
|
||||
monkeypatch.setattr(transcript, "get_webui_dir", lambda: webui_dir)
|
||||
unfinished_key = "websocket:unfinished"
|
||||
completed_key = "websocket:completed"
|
||||
(webui_dir / f"{SessionManager.safe_key(unfinished_key)}.jsonl").write_text(
|
||||
'{"event":"user","chat_id":"unfinished","text":"keep going"}\n'
|
||||
'{"event":"message","chat_id":"unfinished","kind":"progress","text":"Working"}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
(webui_dir / f"{SessionManager.safe_key(completed_key)}.jsonl").write_text(
|
||||
'{"event":"user","chat_id":"completed","text":"done"}\n'
|
||||
'{"event":"message","chat_id":"completed","text":"finished"}\n'
|
||||
'{"event":"turn_end","chat_id":"completed"}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
coordinator, bus, sessions = _coordinator(tmp_path / "workspace")
|
||||
|
||||
await coordinator.scan()
|
||||
|
||||
restored = sessions.get_or_create(unfinished_key)
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "interrupted_without_checkpoint"
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["can_continue"] is False
|
||||
assert sessions.read_session_metadata(completed_key) is None
|
||||
event = bus.outbound.get_nowait().event
|
||||
assert isinstance(event, RecoveryStateEvent)
|
||||
assert event.status == "awaiting_user"
|
||||
assert event.can_continue is False
|
||||
assert bus.outbound.get_nowait().event.scope == "thread"
|
||||
assert bus.outbound.empty()
|
||||
|
||||
with pytest.raises(RecoveryActionError, match="context is unavailable"):
|
||||
await coordinator.handle_action(
|
||||
"continue",
|
||||
{"chat_id": "unfinished", "recovery_id": event.recovery_id},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_loads_only_sessions_that_need_webui_recovery(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
sessions = SessionManager(tmp_path)
|
||||
for key in ("telegram:idle", "discord:pending", "websocket:idle"):
|
||||
session = sessions.get_or_create(key)
|
||||
if key == "discord:pending":
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
_persist(sessions, session)
|
||||
pending = sessions.get_or_create("websocket:pending")
|
||||
pending.metadata[PENDING_USER_TURN_KEY] = True
|
||||
_persist(sessions, pending)
|
||||
|
||||
coordinator, _, restarted = _coordinator(tmp_path)
|
||||
loaded: list[str] = []
|
||||
get_or_create = restarted.get_or_create
|
||||
|
||||
def tracked_get_or_create(key: str) -> Session:
|
||||
loaded.append(key)
|
||||
return get_or_create(key)
|
||||
|
||||
monkeypatch.setattr(restarted, "get_or_create", tracked_get_or_create)
|
||||
monkeypatch.setattr(
|
||||
"nanobot.webui.transcript.has_unfinished_transcript_tail",
|
||||
lambda _key: False,
|
||||
)
|
||||
|
||||
await coordinator.scan()
|
||||
|
||||
assert loaded == ["websocket:pending"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_turn_followup_survives_restart_until_it_is_committed(tmp_path: Path) -> None:
|
||||
"""A message injected mid-turn is not lost between checkpoints."""
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
_persist(sessions, session)
|
||||
followup_id = record_pending_followup(
|
||||
session,
|
||||
InboundMessage(
|
||||
channel="websocket",
|
||||
sender_id="user",
|
||||
chat_id="chat",
|
||||
content="also check the logs",
|
||||
metadata={"webui": True},
|
||||
),
|
||||
)
|
||||
assert followup_id is not None
|
||||
sessions.save(session)
|
||||
|
||||
coordinator, bus, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
queued = bus.inbound.get_nowait()
|
||||
assert queued.content == "also check the logs"
|
||||
assert queued.metadata["_recovery_followup_id"] == followup_id
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
assert len(pending_followups(restored)) == 1
|
||||
|
||||
acknowledge_pending_followups(restored, [followup_id])
|
||||
assert PENDING_FOLLOWUPS_KEY not in restored.metadata
|
||||
|
||||
|
||||
def test_followup_journal_keeps_every_uncommitted_message(tmp_path: Path) -> None:
|
||||
"""A live queue limit must never truncate durable WebUI follow-ups."""
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
followup_ids = [
|
||||
record_pending_followup(
|
||||
session,
|
||||
InboundMessage(
|
||||
channel="websocket",
|
||||
sender_id="user",
|
||||
chat_id="chat",
|
||||
content=f"follow-up-{index}",
|
||||
metadata={"webui": True},
|
||||
),
|
||||
)
|
||||
for index in range(21)
|
||||
]
|
||||
|
||||
assert all(followup_ids)
|
||||
sessions.save(session)
|
||||
restarted = SessionManager(tmp_path)
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
assert [message.content for message in pending_followups(restored)] == [
|
||||
f"follow-up-{index}" for index in range(21)
|
||||
]
|
||||
|
||||
|
||||
def test_requeued_followup_preserves_its_journal_id(tmp_path: Path) -> None:
|
||||
"""Routing a recovered follow-up into a live turn must remain idempotent."""
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
original_id = record_pending_followup(
|
||||
session,
|
||||
InboundMessage(
|
||||
channel="websocket",
|
||||
sender_id="user",
|
||||
chat_id="chat",
|
||||
content="also check the logs",
|
||||
metadata={"webui": True},
|
||||
),
|
||||
)
|
||||
assert original_id is not None
|
||||
|
||||
recovered = pending_followups(session)[0]
|
||||
assert record_pending_followup(session, recovered) == original_id
|
||||
assert [record["id"] for record in session.metadata[PENDING_FOLLOWUPS_KEY]] == [
|
||||
original_id
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_tools_wait_for_confirmation_after_restart(tmp_path: Path) -> None:
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "inspect"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
|
||||
"phase": "tools_completed",
|
||||
"assistant_message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [{"id": "call-1", "function": {"name": "read_file"}}],
|
||||
},
|
||||
"completed_tool_results": [
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call-1",
|
||||
"name": "read_file",
|
||||
"content": "saved result",
|
||||
}
|
||||
],
|
||||
"pending_tool_calls": [],
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, bus, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
assert bus.inbound.empty()
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
assert restored.messages[-1]["content"] == "saved result"
|
||||
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_uncertain_tool_is_never_replayed(tmp_path: Path) -> None:
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "send it"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
|
||||
"phase": "awaiting_tools",
|
||||
"assistant_message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [{"id": "call-1", "function": {"name": "send_email"}}],
|
||||
},
|
||||
"completed_tool_results": [],
|
||||
"pending_tool_calls": [
|
||||
{"id": "call-1", "function": {"name": "send_email"}}
|
||||
],
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, bus, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
assert bus.inbound.empty()
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "tool_state_unknown"
|
||||
assert restored.messages[-1]["_recovery_interrupted"] is True
|
||||
event = bus.outbound.get_nowait().event
|
||||
assert isinstance(event, RecoveryStateEvent)
|
||||
assert event.status == "awaiting_user"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_checkpoint_waits_for_confirmation(tmp_path: Path) -> None:
|
||||
"""Malformed or newer checkpoint phases fail closed pending confirmation."""
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "deploy it"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {"phase": "future_phase"}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, bus, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
assert bus.inbound.empty()
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["status"] == "awaiting_user"
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "checkpoint_unknown"
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["can_continue"] is False
|
||||
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
|
||||
assert PENDING_USER_TURN_KEY not in restored.metadata
|
||||
assert [message["role"] for message in restored.messages] == ["user", "assistant"]
|
||||
assert restored.messages[-1]["_recovery_interrupted"] is True
|
||||
|
||||
with pytest.raises(RecoveryActionError, match="context is unavailable"):
|
||||
await coordinator.handle_action(
|
||||
"continue",
|
||||
{
|
||||
"chat_id": "chat",
|
||||
"recovery_id": restored.metadata[RECOVERY_METADATA_KEY]["recovery_id"],
|
||||
},
|
||||
)
|
||||
|
||||
dismissed = await coordinator.handle_action(
|
||||
"dismiss",
|
||||
{
|
||||
"chat_id": "chat",
|
||||
"recovery_id": restored.metadata[RECOVERY_METADATA_KEY]["recovery_id"],
|
||||
},
|
||||
)
|
||||
assert dismissed["status"] == "recovered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_checkpoint_can_always_be_dismissed(tmp_path: Path) -> None:
|
||||
"""Corrupt private state must not trap the user in a failed recovery notice."""
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "deploy it"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
|
||||
"phase": "future_phase",
|
||||
"assistant_message": "invalid",
|
||||
"completed_tool_results": 3,
|
||||
"pending_tool_calls": [{"id": "call-1", "function": "invalid"}],
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, _, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
state = restored.metadata[RECOVERY_METADATA_KEY]
|
||||
|
||||
result = await coordinator.handle_action(
|
||||
"dismiss",
|
||||
{"chat_id": "chat", "recovery_id": state["recovery_id"]},
|
||||
)
|
||||
|
||||
assert result["status"] == "recovered"
|
||||
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_known_but_malformed_checkpoint_cannot_continue(tmp_path: Path) -> None:
|
||||
"""A known phase does not make corrupt tool state safe to resume."""
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "send it"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
|
||||
"phase": "tools_completed",
|
||||
"assistant_message": {"role": "assistant", "content": "working"},
|
||||
"completed_tool_results": "missing durable results",
|
||||
"pending_tool_calls": [],
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, _, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
state = restored.metadata[RECOVERY_METADATA_KEY]
|
||||
|
||||
assert state["status"] == "awaiting_user"
|
||||
assert state["reason"] == "checkpoint_invalid"
|
||||
assert state["can_continue"] is False
|
||||
with pytest.raises(RecoveryActionError, match="context is unavailable"):
|
||||
await coordinator.handle_action(
|
||||
"continue",
|
||||
{"chat_id": "chat", "recovery_id": state["recovery_id"]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_final_response_is_not_reported_as_restored(tmp_path: Path) -> None:
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "answer"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
|
||||
"phase": "final_response",
|
||||
"assistant_message": "not an answer row",
|
||||
"completed_tool_results": [],
|
||||
"pending_tool_calls": [],
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, _, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
|
||||
assert state["status"] == "awaiting_user"
|
||||
assert state["reason"] == "checkpoint_invalid"
|
||||
assert state["can_continue"] is False
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
|
||||
assert PENDING_USER_TURN_KEY not in restored.metadata
|
||||
assert [message["role"] for message in restored.messages] == ["user", "assistant"]
|
||||
assert restored.messages[-1]["_recovery_interrupted"] is True
|
||||
assert all("tool_calls" not in message for message in restored.messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checkpoint_with_missing_tool_result_cannot_continue(tmp_path: Path) -> None:
|
||||
"""Never resume when persisted results do not cover every requested tool."""
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "send both"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
|
||||
"phase": "tools_completed",
|
||||
"assistant_message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{"id": "call-1", "function": {"name": "send_email"}},
|
||||
{"id": "call-2", "function": {"name": "send_email"}},
|
||||
],
|
||||
},
|
||||
"completed_tool_results": [
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call-1",
|
||||
"name": "send_email",
|
||||
"content": "sent",
|
||||
}
|
||||
],
|
||||
"pending_tool_calls": [],
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, _, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
state = restored.metadata[RECOVERY_METADATA_KEY]
|
||||
assert state["status"] == "awaiting_user"
|
||||
assert state["reason"] == "checkpoint_invalid"
|
||||
assert state["can_continue"] is False
|
||||
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
|
||||
assert PENDING_USER_TURN_KEY not in restored.metadata
|
||||
assert [message["role"] for message in restored.messages] == ["user", "assistant"]
|
||||
assert restored.messages[-1]["_recovery_interrupted"] is True
|
||||
assert all("tool_calls" not in message for message in restored.messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_final_response_is_not_reported_as_restored(tmp_path: Path) -> None:
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "answer"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
|
||||
"phase": "final_response",
|
||||
"assistant_message": {"role": "assistant", "content": ""},
|
||||
"completed_tool_results": [],
|
||||
"pending_tool_calls": [],
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, _, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
|
||||
assert state["status"] == "awaiting_user"
|
||||
assert state["reason"] == "checkpoint_invalid"
|
||||
assert state["can_continue"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_answer_is_restored_without_model_call(tmp_path: Path) -> None:
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "answer"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
|
||||
"phase": "final_response",
|
||||
"assistant_message": {"role": "assistant", "content": "already finished"},
|
||||
"completed_tool_results": [],
|
||||
"pending_tool_calls": [],
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, bus, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
assert bus.inbound.empty()
|
||||
restored = restarted.get_or_create("websocket:chat")
|
||||
assert restored.messages[-1]["content"] == "already finished"
|
||||
first = bus.outbound.get_nowait().event
|
||||
second = bus.outbound.get_nowait().event
|
||||
assert isinstance(first, RecoveryStateEvent) and first.status == "recovered"
|
||||
assert isinstance(second, SessionUpdatedEvent)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explicit_recovery_continue_queues_once(tmp_path: Path) -> None:
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "continue"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, bus, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
assert bus.inbound.empty()
|
||||
|
||||
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
|
||||
result = await coordinator.handle_action(
|
||||
"continue",
|
||||
{"chat_id": "chat", "recovery_id": state["recovery_id"]},
|
||||
)
|
||||
assert result["status"] == "resuming"
|
||||
assert bus.inbound.get_nowait().metadata["_webui_recovery_id"] == state["recovery_id"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_user_message_supersedes_waiting_recovery(tmp_path: Path) -> None:
|
||||
coordinator, bus, sessions = _coordinator(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.append({"role": "user", "content": "old request"})
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
_persist(sessions, session)
|
||||
await coordinator.scan()
|
||||
|
||||
newer = InboundMessage(
|
||||
channel="websocket",
|
||||
sender_id="user",
|
||||
chat_id="chat",
|
||||
content="new request",
|
||||
)
|
||||
assert await coordinator.admit(newer) is True
|
||||
restored = sessions.get_or_create("websocket:chat")
|
||||
assert restored.messages[-1]["_recovery_interrupted"] is True
|
||||
assert sum(
|
||||
message.get("_recovery_interrupted") is True
|
||||
for message in restored.messages
|
||||
) == 1
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "superseded"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_user_message_cancels_active_recovery_task(tmp_path: Path) -> None:
|
||||
coordinator, bus, sessions = _coordinator(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.metadata[RECOVERY_METADATA_KEY] = {
|
||||
"status": "resuming",
|
||||
"recovery_id": "active",
|
||||
"attempts": 1,
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
started = asyncio.Event()
|
||||
|
||||
async def _active_recovery() -> None:
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
task = asyncio.create_task(_active_recovery())
|
||||
await started.wait()
|
||||
coordinator.register_recovery_task("websocket:chat", task)
|
||||
|
||||
newer = InboundMessage(
|
||||
channel="websocket",
|
||||
sender_id="user",
|
||||
chat_id="chat",
|
||||
content="new request",
|
||||
)
|
||||
assert await coordinator.admit(newer) is True
|
||||
assert task.cancelled()
|
||||
restored = sessions.get_or_create("websocket:chat")
|
||||
assert restored.metadata[RECOVERY_METADATA_KEY]["reason"] == "superseded"
|
||||
assert restored.messages[-1]["_recovery_interrupted"] is True
|
||||
first = bus.outbound.get_nowait().event
|
||||
second = bus.outbound.get_nowait().event
|
||||
assert isinstance(first, RecoveryStateEvent)
|
||||
assert isinstance(second, SessionUpdatedEvent)
|
||||
assert bus.outbound.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recovery_action_rejects_stale_page_and_continues_current_state(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
coordinator, bus, sessions = _coordinator(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.metadata[RECOVERY_METADATA_KEY] = {
|
||||
"status": "awaiting_user",
|
||||
"recovery_id": "current",
|
||||
"attempts": 0,
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
with pytest.raises(RecoveryActionError, match="stale"):
|
||||
await coordinator.handle_action(
|
||||
"continue",
|
||||
{"chat_id": "chat", "recovery_id": "old"},
|
||||
)
|
||||
|
||||
result = await coordinator.handle_action(
|
||||
"continue",
|
||||
{"chat_id": "chat", "recovery_id": "current"},
|
||||
)
|
||||
assert result["status"] == "resuming"
|
||||
assert bus.inbound.get_nowait().metadata["_webui_recovery_id"] == "current"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_persisted_completion_wins_over_stale_resuming_marker(tmp_path: Path) -> None:
|
||||
sessions = SessionManager(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.messages.extend(
|
||||
[
|
||||
{"role": "user", "content": "work"},
|
||||
{"role": "assistant", "content": "done"},
|
||||
]
|
||||
)
|
||||
session.metadata[RECOVERY_METADATA_KEY] = {
|
||||
"status": "resuming",
|
||||
"recovery_id": "recovery",
|
||||
"attempts": 1,
|
||||
"resume_message_count": 1,
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
coordinator, bus, restarted = _coordinator(tmp_path)
|
||||
await coordinator.scan()
|
||||
|
||||
assert bus.inbound.empty()
|
||||
state = restarted.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
|
||||
assert state["status"] == "recovered"
|
||||
assert state["reason"] == "committed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dismiss_does_not_queue_work(tmp_path: Path) -> None:
|
||||
coordinator, bus, sessions = _coordinator(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.metadata[RECOVERY_METADATA_KEY] = {
|
||||
"status": "awaiting_user",
|
||||
"recovery_id": "current",
|
||||
"attempts": 0,
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
result = await coordinator.handle_action(
|
||||
"dismiss",
|
||||
{"chat_id": "chat", "recovery_id": "current"},
|
||||
)
|
||||
|
||||
assert result["status"] == "recovered"
|
||||
assert bus.inbound.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_failure_is_visible_instead_of_aborting_other_sessions(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
coordinator, bus, sessions = _coordinator(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
_persist(sessions, session)
|
||||
|
||||
async def fail(*_args: object, **_kwargs: object) -> None:
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(RecoveryCoordinator, "_recover_session", fail)
|
||||
await coordinator.scan()
|
||||
|
||||
state = sessions.get_or_create("websocket:chat").metadata[RECOVERY_METADATA_KEY]
|
||||
assert state["status"] == "failed"
|
||||
assert state["can_continue"] is False
|
||||
assert isinstance(bus.outbound.get_nowait().event, RecoveryStateEvent)
|
||||
|
||||
with pytest.raises(RecoveryActionError, match="context is unavailable"):
|
||||
await coordinator.handle_action(
|
||||
"continue",
|
||||
{"chat_id": "chat", "recovery_id": state["recovery_id"]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bus_remains_quiet_after_recovered_state(tmp_path: Path) -> None:
|
||||
coordinator, bus, sessions = _coordinator(tmp_path)
|
||||
session = sessions.get_or_create("websocket:chat")
|
||||
session.metadata[RECOVERY_METADATA_KEY] = {
|
||||
"status": "recovered",
|
||||
"recovery_id": "done",
|
||||
"attempts": 1,
|
||||
}
|
||||
_persist(sessions, session)
|
||||
|
||||
await coordinator.scan()
|
||||
|
||||
await asyncio.sleep(0)
|
||||
assert bus.inbound.empty()
|
||||
assert bus.outbound.empty()
|
||||
@@ -1,6 +1,7 @@
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import nanobot.session as session_api
|
||||
from nanobot.providers.base import ProviderConversationState
|
||||
from nanobot.session import Session, SessionManager
|
||||
from nanobot.session.manager import SessionStore
|
||||
from nanobot.session.model_selection import SESSION_MODEL_PRESET_METADATA_KEY
|
||||
@@ -123,3 +124,93 @@ def test_manager_preserves_full_session_before_store_save(tmp_path) -> None:
|
||||
assert session.messages[0]["content"] == "0"
|
||||
assert session.messages[-1]["content"] == "2000"
|
||||
store.save.assert_called_once_with(session, fsync=False)
|
||||
|
||||
|
||||
def test_runtime_checkpoint_does_not_rewrite_long_session(tmp_path) -> None:
|
||||
manager = SessionManager(tmp_path)
|
||||
session = manager.get_or_create("websocket:long")
|
||||
for index in range(256):
|
||||
session.add_message("user", f"{index}:" + "x" * 4096)
|
||||
manager.save(session)
|
||||
|
||||
main_path = manager._get_session_path(session.key)
|
||||
main_before = main_path.read_bytes()
|
||||
stat_before = main_path.stat()
|
||||
session.metadata["runtime_checkpoint"] = {
|
||||
"phase": "tools_completed",
|
||||
"assistant_message": {"role": "assistant", "content": "working"},
|
||||
"completed_tool_results": [],
|
||||
"pending_tool_calls": [],
|
||||
}
|
||||
session.provider_state = ProviderConversationState(
|
||||
kind="openai_responses",
|
||||
provider="openai:test",
|
||||
model="test-model",
|
||||
version=1,
|
||||
payload={"response_id": "private-response"},
|
||||
)
|
||||
|
||||
manager.save_runtime_checkpoint(session)
|
||||
|
||||
checkpoint_path = manager._get_runtime_checkpoint_path(session.key)
|
||||
assert main_path.read_bytes() == main_before
|
||||
assert main_path.stat().st_ino == stat_before.st_ino
|
||||
assert main_path.stat().st_mtime_ns == stat_before.st_mtime_ns
|
||||
assert checkpoint_path.stat().st_size < len(main_before) // 100
|
||||
|
||||
restored = SessionManager(tmp_path).get_or_create(session.key)
|
||||
assert restored.metadata["runtime_checkpoint"]["phase"] == "tools_completed"
|
||||
assert restored.provider_state is not None
|
||||
assert restored.provider_state.payload == {"response_id": "private-response"}
|
||||
|
||||
|
||||
def test_completed_session_supersedes_stale_checkpoint(tmp_path) -> None:
|
||||
manager = SessionManager(tmp_path)
|
||||
session = manager.get_or_create("websocket:completed")
|
||||
session.add_message("user", "question")
|
||||
manager.save(session)
|
||||
session.metadata["runtime_checkpoint"] = {"phase": "awaiting_tools"}
|
||||
manager.save_runtime_checkpoint(session)
|
||||
checkpoint_path = manager._get_runtime_checkpoint_path(session.key)
|
||||
stale_checkpoint = checkpoint_path.read_bytes()
|
||||
|
||||
session.metadata.pop("runtime_checkpoint")
|
||||
session.add_message("assistant", "answer")
|
||||
manager.save(session)
|
||||
assert not checkpoint_path.exists()
|
||||
|
||||
# Emulate a process dying after the main record was committed but before an
|
||||
# obsolete sidecar could be removed. The base fingerprint keeps it stale.
|
||||
checkpoint_path.write_bytes(stale_checkpoint)
|
||||
restored = SessionManager(tmp_path).get_or_create(session.key)
|
||||
assert "runtime_checkpoint" not in restored.metadata
|
||||
assert restored.messages[-1]["content"] == "answer"
|
||||
assert not checkpoint_path.exists()
|
||||
|
||||
|
||||
def test_delete_session_removes_runtime_checkpoint(tmp_path) -> None:
|
||||
manager = SessionManager(tmp_path)
|
||||
session = manager.get_or_create("websocket:delete")
|
||||
session.add_message("user", "question")
|
||||
manager.save(session)
|
||||
session.metadata["runtime_checkpoint"] = {"phase": "awaiting_tools"}
|
||||
manager.save_runtime_checkpoint(session)
|
||||
checkpoint_path = manager._get_runtime_checkpoint_path(session.key)
|
||||
assert checkpoint_path.exists()
|
||||
|
||||
assert manager.delete_session(session.key) is True
|
||||
assert not checkpoint_path.exists()
|
||||
|
||||
|
||||
def test_invalid_runtime_checkpoint_is_discarded(tmp_path) -> None:
|
||||
manager = SessionManager(tmp_path)
|
||||
session = manager.get_or_create("websocket:invalid-checkpoint")
|
||||
session.add_message("user", "question")
|
||||
manager.save(session)
|
||||
checkpoint_path = manager._get_runtime_checkpoint_path(session.key)
|
||||
checkpoint_path.write_text("{truncated", encoding="utf-8")
|
||||
|
||||
restored = SessionManager(tmp_path).get_or_create(session.key)
|
||||
|
||||
assert "runtime_checkpoint" not in restored.metadata
|
||||
assert not checkpoint_path.exists()
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import nanobot.webui.transcript as transcript_module
|
||||
from nanobot.session.history_visibility import HIDDEN_HISTORY_META
|
||||
from nanobot.webui.transcript import (
|
||||
@@ -554,6 +556,22 @@ def test_thread_response_marks_unfinished_tool_tail_pending(tmp_path, monkeypatc
|
||||
assert out["completed_turn_ids"] == []
|
||||
|
||||
|
||||
def test_recovery_tail_check_reads_only_the_active_transcript(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
|
||||
key = "websocket:recovery-tail"
|
||||
active_path = transcript_module.webui_transcript_path(key)
|
||||
reads: list[Path] = []
|
||||
|
||||
def read(path: Path) -> list[dict[str, object]]:
|
||||
reads.append(path)
|
||||
return [{"event": "message", "kind": "progress", "text": "running"}]
|
||||
|
||||
monkeypatch.setattr(transcript_module, "_read_transcript_file", read)
|
||||
|
||||
assert transcript_module.has_unfinished_transcript_tail(key) is True
|
||||
assert reads == [active_path]
|
||||
|
||||
|
||||
def test_thread_response_reports_active_registry_without_transcript(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
|
||||
@@ -15,6 +15,9 @@ import httpx
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
from nanobot.session.manager import SessionManager
|
||||
from nanobot.session.recovery import PENDING_USER_TURN_KEY, RUNTIME_CHECKPOINT_KEY
|
||||
|
||||
_BOOTSTRAP_SECRET = "smoke-secret"
|
||||
|
||||
|
||||
@@ -206,3 +209,84 @@ async def test_gateway_webui_bootstrap_message_and_thread_hydration(tmp_path: Pa
|
||||
assert any("shell-ok" in text for text in contents)
|
||||
finally:
|
||||
_stop_gateway(process)
|
||||
|
||||
|
||||
def test_gateway_restart_restores_a_completed_answer_without_replaying_model(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Exercise recovery through two real gateway processes and durable files."""
|
||||
ws_port = _free_port()
|
||||
gateway_port = _free_port()
|
||||
workspace = tmp_path / "workspace"
|
||||
workspace.mkdir()
|
||||
config_path = tmp_path / "config.json"
|
||||
first_log = tmp_path / "gateway-first.log"
|
||||
second_log = tmp_path / "gateway-second.log"
|
||||
_write_smoke_config(
|
||||
config_path,
|
||||
workspace=workspace,
|
||||
ws_port=ws_port,
|
||||
gateway_port=gateway_port,
|
||||
)
|
||||
base_url = f"http://127.0.0.1:{ws_port}"
|
||||
|
||||
first = _start_gateway(config_path, first_log)
|
||||
try:
|
||||
_wait_for_bootstrap(base_url, first, first_log)
|
||||
finally:
|
||||
_stop_gateway(first)
|
||||
|
||||
sessions_root = tmp_path / "sessions"
|
||||
sessions = SessionManager(workspace, sessions_root=sessions_root)
|
||||
session = sessions.get_or_create("websocket:recovery-smoke")
|
||||
session.messages.append({"role": "user", "content": "recover this answer"})
|
||||
session.metadata["webui"] = True
|
||||
session.metadata[PENDING_USER_TURN_KEY] = True
|
||||
session.metadata[RUNTIME_CHECKPOINT_KEY] = {
|
||||
"phase": "final_response",
|
||||
"assistant_message": {
|
||||
"role": "assistant",
|
||||
"content": "restored without another model request",
|
||||
},
|
||||
"completed_tool_results": [],
|
||||
"pending_tool_calls": [],
|
||||
}
|
||||
sessions.save(session, fsync=True)
|
||||
|
||||
second = _start_gateway(config_path, second_log)
|
||||
try:
|
||||
bootstrap = _wait_for_bootstrap(base_url, second, second_log)
|
||||
deadline = time.monotonic() + 20
|
||||
restored = None
|
||||
while time.monotonic() < deadline:
|
||||
restored = SessionManager(
|
||||
workspace,
|
||||
sessions_root=sessions_root,
|
||||
).get_or_create("websocket:recovery-smoke")
|
||||
if any(
|
||||
message.get("content") == "restored without another model request"
|
||||
for message in restored.messages
|
||||
):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
logs = second_log.read_text(encoding="utf-8", errors="replace")
|
||||
raise AssertionError(f"answer was not recovered after restart\n{logs}")
|
||||
|
||||
assert restored is not None
|
||||
assert PENDING_USER_TURN_KEY not in restored.metadata
|
||||
assert RUNTIME_CHECKPOINT_KEY not in restored.metadata
|
||||
assert restored.metadata["webui_recovery"]["reason"] == "answer_restored"
|
||||
|
||||
async def assert_attach_state() -> None:
|
||||
ws_url = f'{bootstrap["ws_url"]}?token={bootstrap["token"]}&client_id=recovery-smoke'
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
await _recv_until(ws, "ready")
|
||||
await ws.send(json.dumps({"type": "attach", "chat_id": "recovery-smoke"}))
|
||||
attached = await _recv_until(ws, "attached")
|
||||
assert attached["recovery_state"]["status"] == "recovered"
|
||||
assert attached["recovery_state"]["reason"] == "answer_restored"
|
||||
|
||||
asyncio.run(assert_attach_state())
|
||||
finally:
|
||||
_stop_gateway(second)
|
||||
|
||||
@@ -18,6 +18,7 @@ from nanobot.session.automation_turns import AUTOMATION_HISTORY_META
|
||||
from nanobot.session.history_visibility import HIDDEN_HISTORY_META
|
||||
from nanobot.session.manager import SessionManager
|
||||
from nanobot.session.model_selection import SESSION_MODEL_PRESET_METADATA_KEY
|
||||
from nanobot.session.recovery import RECOVERY_METADATA_KEY
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -66,6 +67,30 @@ def test_webui_session_list_refreshes_after_model_preset_rename(tmp_path: Path)
|
||||
assert list_webui_sessions(manager)[0]["model_preset"] == "Codex"
|
||||
|
||||
|
||||
def test_webui_session_list_surfaces_pending_recovery_state(tmp_path: Path) -> None:
|
||||
manager = SessionManager(tmp_path)
|
||||
session = manager.get_or_create("websocket:needs-attention")
|
||||
session.add_message("user", "the interrupted task")
|
||||
session.metadata[RECOVERY_METADATA_KEY] = {
|
||||
"status": "awaiting_user",
|
||||
"recovery_id": "recovery-123",
|
||||
"reason": "uncertain_tool_state",
|
||||
"attempts": 1,
|
||||
# Private checkpoint details must never leak into the sidebar index.
|
||||
"checkpoint": {"tool_args": "secret"},
|
||||
}
|
||||
manager.save(session)
|
||||
|
||||
row = list_webui_sessions(manager)[0]
|
||||
|
||||
assert row["recovery_state"] == {
|
||||
"status": "awaiting_user",
|
||||
"recovery_id": "recovery-123",
|
||||
"reason": "uncertain_tool_state",
|
||||
"attempts": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_webui_session_index_uses_unique_temp_file(tmp_path: Path) -> None:
|
||||
manager = SessionManager(tmp_path)
|
||||
session = manager.get_or_create("websocket:unique-index-temp")
|
||||
|
||||
Reference in New Issue
Block a user