perf(webui): accelerate JSONL session list and thread loading (#5194)

tests/utils/test_webui_thread_disk.py

@@ -13,6 +13,7 @@ from nanobot.webui.transcript import (
 def test_delete_webui_thread_removes_legacy_json_and_transcript(tmp_path, monkeypatch) -> None:
     monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
     monkeypatch.setattr("nanobot.webui.transcript._MAX_TRANSCRIPT_FILE_BYTES", 520)
+    monkeypatch.setattr("nanobot.webui.transcript._ACTIVE_TRANSCRIPT_ROTATE_BYTES", 520)
     monkeypatch.setattr("nanobot.webui.transcript._TARGET_ACTIVE_TRANSCRIPT_BYTES", 260)
     key = "websocket:k1"
     json_path = webui_thread_file_path(key)

tests/utils/test_webui_transcript.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import nanobot.webui.transcript as transcript_module
 from nanobot.session.history_visibility import HIDDEN_HISTORY_META
 from nanobot.webui.transcript import (
     WEBUI_TRANSCRIPT_SCHEMA_VERSION,
@@ -38,6 +39,7 @@ def test_append_stamps_created_at_ms(tmp_path, monkeypatch) -> None:
 
 def _force_small_transcript_budget(monkeypatch, *, limit: int = 520, target: int = 260) -> None:
     monkeypatch.setattr("nanobot.webui.transcript._MAX_TRANSCRIPT_FILE_BYTES", limit)
+    monkeypatch.setattr("nanobot.webui.transcript._ACTIVE_TRANSCRIPT_ROTATE_BYTES", limit)
     monkeypatch.setattr("nanobot.webui.transcript._TARGET_ACTIVE_TRANSCRIPT_BYTES", target)
 
 
@@ -122,6 +124,28 @@ def test_segmented_transcript_paginates_latest_and_older_without_overlap(
     ]
 
 
+def test_latest_page_reads_active_chunk_once(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
+    key = "websocket:single-active-read"
+    for idx in range(1, 7):
+        _append_numbered_turn(key, "single-active-read", idx)
+
+    original = transcript_module._read_chunk_turns
+    read_chunk_ids: list[str] = []
+
+    def track_read(session_key: str, chunk_id: str) -> list[list[dict]]:
+        read_chunk_ids.append(chunk_id)
+        return original(session_key, chunk_id)
+
+    monkeypatch.setattr(transcript_module, "_read_chunk_turns", track_read)
+
+    latest = build_webui_thread_response(key, limit=4, direction="latest")
+
+    assert latest is not None
+    assert _message_contents(latest) == _numbered_turn_texts(5, 6)
+    assert read_chunk_ids == ["active"]
+
+
 def test_page_cursor_survives_active_rotation_after_latest_page(
     tmp_path,
     monkeypatch,
@@ -148,15 +172,53 @@ def test_segment_manifest_can_be_rebuilt_when_missing_or_corrupt(tmp_path, monke
     key = "websocket:manifest"
     _write_segmented_turns(tmp_path, monkeypatch, key, "manifest", 4)
 
-    manifest = webui_transcript_segments_dir(key) / "manifest.json"
+    segment_dir = webui_transcript_segments_dir(key)
+    segment_names = sorted(path.name for path in segment_dir.glob("*.jsonl"))
+    assert segment_names
+    original = transcript_module._read_transcript_file
+    segment_reads: list[str] = []
+
+    def track_read(path):
+        if path.parent == segment_dir and path.suffix == ".jsonl":
+            segment_reads.append(path.name)
+        return original(path)
+
+    monkeypatch.setattr(transcript_module, "_read_transcript_file", track_read)
+    manifest = segment_dir / "manifest.json"
     manifest.write_text("{not json", encoding="utf-8")
 
+    entries = transcript_module._read_segment_manifest_entries(key)
+
+    assert [entry["id"] for entry in entries] == [path.removesuffix(".jsonl") for path in segment_names]
+    assert segment_reads == segment_names
+
     lines = read_transcript_lines(key)
 
     assert len([line for line in lines if line.get("event") == "user"]) == 4
     assert manifest.read_text(encoding="utf-8").lstrip().startswith("{")
 
 
+def test_rotation_does_not_reread_existing_segments(tmp_path, monkeypatch) -> None:
+    key = "websocket:manifest-append"
+    _write_segmented_turns(tmp_path, monkeypatch, key, "manifest-append", 4)
+    segment_dir = webui_transcript_segments_dir(key)
+    assert list(segment_dir.glob("*.jsonl"))
+
+    original = transcript_module._read_transcript_file
+    segment_reads: list[str] = []
+
+    def track_read(path):
+        if path.parent == segment_dir and path.suffix == ".jsonl":
+            segment_reads.append(path.name)
+        return original(path)
+
+    monkeypatch.setattr(transcript_module, "_read_transcript_file", track_read)
+    for idx in range(5, 9):
+        _append_numbered_turn(key, "manifest-append", idx)
+
+    assert segment_reads == []
+
+
 def test_delete_webui_transcript_removes_segments(tmp_path, monkeypatch) -> None:
     from nanobot.webui.thread_disk import webui_thread_file_path
     from nanobot.webui.transcript import delete_webui_transcript, webui_transcript_path
@@ -786,6 +848,83 @@ def test_build_response_restores_session_users_for_legacy_transcript(
     ]
 
 
+def test_complete_transcript_does_not_load_session_messages(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
+    key = "websocket:complete-fast-path"
+    for event in (
+        {"event": "user", "chat_id": "complete-fast-path", "text": "question"},
+        {"event": "message", "chat_id": "complete-fast-path", "text": "answer"},
+        {"event": "turn_end", "chat_id": "complete-fast-path"},
+    ):
+        append_transcript_object(key, event)
+
+    def fail_if_loaded() -> list[dict]:
+        raise AssertionError("complete transcripts must not read canonical session history")
+
+    out = build_webui_thread_response(
+        key,
+        limit=4,
+        direction="latest",
+        session_messages_loader=fail_if_loaded,
+    )
+
+    assert out is not None
+    assert [(message["role"], message["content"]) for message in out["messages"]] == [
+        ("user", "question"),
+        ("assistant", "answer"),
+    ]
+
+
+def test_legacy_recovery_loads_session_and_builds_backfill_turns_once(
+    tmp_path,
+    monkeypatch,
+) -> None:
+    monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
+    key = "websocket:lazy-legacy-recovery"
+    append_transcript_object(
+        key,
+        {"event": "message", "chat_id": "lazy-legacy-recovery", "text": "answer"},
+    )
+    append_transcript_object(
+        key,
+        {
+            "event": "turn_end",
+            "chat_id": "lazy-legacy-recovery",
+            "transcript_incomplete": True,
+        },
+    )
+
+    loader_calls = 0
+    backfill_calls = 0
+    original = transcript_module._session_backfill_turns
+
+    def load_session_messages() -> list[dict]:
+        nonlocal loader_calls
+        loader_calls += 1
+        return [
+            {"role": "user", "content": "question"},
+            {"role": "assistant", "content": "answer"},
+        ]
+
+    def track_backfill(session_key: str, session_messages: list[dict]):
+        nonlocal backfill_calls
+        backfill_calls += 1
+        return original(session_key, session_messages)
+
+    monkeypatch.setattr(transcript_module, "_session_backfill_turns", track_backfill)
+
+    out = build_webui_thread_response(key, session_messages_loader=load_session_messages)
+
+    assert out is not None
+    assert loader_calls == 1
+    assert backfill_calls == 1
+    assert [(message["role"], message["content"]) for message in out["messages"]] == [
+        ("user", "question"),
+        ("assistant", "answer"),
+    ]
+    assert out["has_pending_tool_calls"] is False
+
+
 def test_build_response_restores_session_users_without_duplicating_new_transcript_users(
     tmp_path,
     monkeypatch,

tests/utils/test_webui_workspaces.py

@@ -1,9 +1,14 @@
 import json
+from unittest.mock import MagicMock
 
 import pytest
 
-from nanobot.security.workspace_access import WorkspaceScopeError, default_workspace_scope
-from nanobot.session.manager import SessionManager
+from nanobot.security.workspace_access import (
+    WORKSPACE_SCOPE_METADATA_KEY,
+    WorkspaceScopeError,
+    default_workspace_scope,
+)
+from nanobot.session.manager import SessionManager, SessionStore
 from nanobot.webui.workspaces import (
     WebUIWorkspaceController,
     read_webui_default_access_mode,
@@ -135,6 +140,33 @@ def test_webui_default_access_applies_to_unscoped_old_sessions(tmp_path, monkeyp
     assert new_scope.access_mode == "full"
 
 
+def test_indexed_scope_preserves_missing_and_explicit_null_semantics(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.webui.workspaces.get_webui_dir", lambda: tmp_path / "webui")
+    default = tmp_path / "default"
+    default.mkdir()
+    write_webui_default_access_mode("full")
+    controller = WebUIWorkspaceController(
+        session_manager=None,
+        default_workspace=default,
+        default_restrict_to_workspace=True,
+    )
+    webui_default = controller.default_scope()
+
+    missing = controller.scope_for_indexed_metadata(
+        None,
+        scope_present=False,
+        default_scope=webui_default,
+    )
+    explicit_null = controller.scope_for_indexed_metadata(
+        None,
+        scope_present=True,
+        default_scope=webui_default,
+    )
+
+    assert missing.access_mode == "full"
+    assert explicit_null.access_mode == "restricted"
+
+
 def test_webui_default_access_does_not_override_explicit_session_scope(tmp_path, monkeypatch) -> None:
     monkeypatch.setattr("nanobot.webui.workspaces.get_webui_dir", lambda: tmp_path / "webui")
     default = tmp_path / "default"
@@ -185,6 +217,53 @@ def test_scope_for_session_key_reads_metadata_without_full_history(
     assert scope.access_mode == "full"
 
 
+def test_scope_for_session_key_always_reads_the_active_store(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.webui.workspaces.get_webui_dir", lambda: tmp_path / "webui")
+    default = tmp_path / "default"
+    project = tmp_path / "project"
+    default.mkdir()
+    project.mkdir()
+    workspace = tmp_path / "session-data"
+    full_scope = default_workspace_scope(project, restrict_to_workspace=False)
+    restricted_scope = default_workspace_scope(project, restrict_to_workspace=True)
+
+    residual_sessions = SessionManager(workspace)
+    residual = residual_sessions.get_or_create("websocket:cached")
+    residual.metadata[WORKSPACE_SCOPE_METADATA_KEY] = full_scope.metadata()
+    residual_sessions.save(residual)
+
+    store = MagicMock(spec=SessionStore)
+    store.read_metadata.side_effect = [
+        {
+            "key": "websocket:cached",
+            "created_at": None,
+            "updated_at": None,
+            "metadata": {WORKSPACE_SCOPE_METADATA_KEY: full_scope.metadata()},
+        },
+        {
+            "key": "websocket:cached",
+            "created_at": None,
+            "updated_at": None,
+            "metadata": {WORKSPACE_SCOPE_METADATA_KEY: restricted_scope.metadata()},
+        },
+    ]
+    sessions = SessionManager(workspace, store=store)
+    controller = WebUIWorkspaceController(
+        session_manager=sessions,
+        default_workspace=default,
+        default_restrict_to_workspace=True,
+    )
+
+    first = controller.scope_for_session_key("websocket:cached")
+    second = controller.scope_for_session_key("websocket:cached")
+
+    assert first.project_path == project.resolve()
+    assert first.access_mode == "full"
+    assert second.project_path == project.resolve()
+    assert second.access_mode == "restricted"
+    assert store.read_metadata.call_count == 2
+
+
 def test_remote_existing_chat_can_reduce_its_workspace_access(tmp_path, monkeypatch) -> None:
     monkeypatch.setattr("nanobot.webui.workspaces.get_webui_dir", lambda: tmp_path / "webui")
     default = tmp_path / "default"