refactor(agent): make memory summaries cumulative (#5610)

* refactor(agent): make memory summaries cumulative

Treat the latest session summary as a replacement checkpoint, preserve it through bounded raw fallbacks, and reserve history.jsonl for Dream ingestion.

* fix(agent): preserve cumulative checkpoint context

* fix(agent): preserve memory archive prompt cache

* refactor(agent): state archive prompt positively

* refactor(agent): remove checkpoint version migration

* refactor(agent): summarize full archive context

* test(agent): align cumulative archive prompt assertion

* refactor(agent): clarify memory checkpoint contract

tests/agent/test_consolidator.py

@@ -1,4 +1,4 @@
-"""Tests for the lightweight Consolidator — append-only to HISTORY.md."""
+"""Tests for Memory checkpoint consolidation and history journaling."""
 
 from dataclasses import replace
 from unittest.mock import AsyncMock, MagicMock
@@ -6,7 +6,7 @@ from unittest.mock import AsyncMock, MagicMock
 import pytest
 
 from nanobot.agent.memory import (
-    _ARCHIVE_SUMMARY_MAX_CHARS,
+    _HISTORY_ENTRY_HARD_CAP,
     Consolidator,
     MemoryStore,
 )
@@ -26,6 +26,8 @@ from nanobot.session.manager import Session
 from nanobot.utils.llm_runtime import LLMRuntime
 from nanobot.utils.prompt_templates import render_template
 
+_ARCHIVE_PROMPT = render_template("agent/consolidator_archive.md", strip=True)
+
 
 @pytest.fixture
 def store(tmp_path):
@@ -98,8 +100,15 @@ def _build_test_messages(**kwargs):
     ]
 
 
-async def _archive(consolidator, messages, runtime, *, session_key="test:session"):
-    return await consolidator.archive(
+async def _archive(
+    consolidator,
+    messages,
+    runtime,
+    *,
+    session_key="test:session",
+    previous_summary=None,
+):
+    return await consolidator.archiver.archive(
         messages,
         runtime=runtime,
         session_key=session_key,
@@ -108,6 +117,7 @@ async def _archive(consolidator, messages, runtime, *, session_key="test:session
             current_message="consolidate",
         ),
         request_tools=[],
+        previous_summary=previous_summary,
     )
 
 
@@ -201,7 +211,9 @@ class TestConsolidatorSummarize:
         mock_provider.chat_with_retry.side_effect = Exception("API error")
         messages = [{"role": "user", "content": "hello"}]
         result = await _archive(consolidator, messages, runtime)
-        assert result is None  # no summary on raw dump fallback
+        assert result is not None
+        assert "[RAW]" in result
+        assert "hello" in result
         entries = store.read_unprocessed_history(since_cursor=0)
         assert len(entries) == 1
         assert "[RAW]" in entries[0]["content"]
@@ -226,23 +238,51 @@ class TestConsolidatorSummarize:
         entries = store.read_unprocessed_history(since_cursor=0)
         assert entries[0]["session_key"] == "slack:chat-2"
 
+    async def test_raw_fallback_represents_previous_checkpoint_and_new_chunk(
+        self,
+        consolidator,
+        mock_provider,
+        runtime,
+    ):
+        runtime = replace(runtime, generation=GenerationSettings(max_tokens=96))
+        mock_provider.chat_with_retry.side_effect = RuntimeError("API error")
+
+        result = await _archive(
+            consolidator,
+            [{"role": "user", "content": "NEW_MARKER " + "new " * 200}],
+            runtime,
+            previous_summary="OLD_MARKER " + "old " * 200,
+        )
+
+        assert result is not None
+        assert "[Previous archived context]" in result
+        assert "OLD_MARKER" in result
+        assert "[Newly archived raw context]" in result
+        assert "NEW_MARKER" in result
+        assert "... (truncated)" in result
+
     async def test_summarize_skips_empty_messages(self, consolidator, runtime):
         result = await _archive(consolidator, [], runtime)
         assert result is None
 
 
 class TestConsolidatorPromptContract:
-    def test_archive_prompt_preserves_working_state_with_memory_facts(self):
-        prompt = render_template("agent/consolidator_archive.md", strip=True, archive_count=4)
+    def test_archive_prompt_requests_a_cumulative_replacement_checkpoint(self):
+        prompt = _ARCHIVE_PROMPT
 
+        for section in ("## Merge rules", "## What to retain", "## Output"):
+            assert section in prompt
+        assert "replacement checkpoint" in prompt
+        assert "[Archived Context Summary]" in prompt
+        assert "current conversation state" in prompt
         assert "SNIP" in prompt
-        assert "final 4 conversation messages" in prompt
-        for mark in ("[permanent]", "[durable]", "[ephemeral]", "[correction]", "[skip]"):
+        for mark in ("[permanent]", "[durable]", "[ephemeral]", "[correction]"):
             assert mark in prompt
         assert "working-state handoff" in prompt
-        assert "exact identifiers needed to continue without rework" in prompt
-        assert "Do not output facts already present in the system prompt's Recent History" in prompt
-        assert "Do not mark something [skip] merely because it might already exist" in prompt
+        assert "- [mark] fact" in prompt
+        assert "[skip]" not in prompt
+        assert "(nothing)" in prompt
+        assert "history.jsonl" not in prompt
 
 
 class TestConsolidatorArchiveErrorHandling:
@@ -272,7 +312,8 @@ class TestConsolidatorArchiveErrorHandling:
             {"role": "assistant", "content": "Done, fixed the race condition."},
         ]
         result = await _archive(consolidator, messages, runtime)
-        assert result is None
+        assert result is not None
+        assert "[RAW]" in result
         entries = store.read_unprocessed_history(since_cursor=0)
         assert len(entries) == 1
         assert "[RAW]" in entries[0]["content"]
@@ -436,9 +477,9 @@ class TestConsolidatorTokenBudget:
         assert [message["content"] for message in request["messages"][1:-1]] == [
             f"m{i}" for i in range(50)
         ]
-        assert "final 50 conversation messages" in request["messages"][-1]["content"]
+        assert request["messages"][-1]["content"] == _ARCHIVE_PROMPT
         assert request["tools"] == []
-        assert request["tool_choice"] == "none"
+        assert "tool_choice" not in request
         assert session.last_archived == 50
         assert session.provider_state == _provider_state()
 
@@ -460,8 +501,7 @@ class TestConsolidatorTokenBudget:
         consolidator.estimate_session_prompt_tokens = MagicMock(
             side_effect=[(1200, "tiktoken"), (400, "tiktoken")]
         )
-        # LLM consolidation fails after raw_archive fires.
-        consolidator.archive_session = AsyncMock(return_value=None)
+        consolidator.archive_session = AsyncMock(return_value="[RAW] checkpoint")
 
         await consolidator.maybe_consolidate_by_tokens(session, runtime=runtime)
 
@@ -491,7 +531,7 @@ class TestConsolidatorTokenBudget:
         consolidator.estimate_session_prompt_tokens = MagicMock(
             return_value=(1200, "tiktoken")
         )
-        consolidator.archive_session = AsyncMock(return_value=None)
+        consolidator.archive_session = AsyncMock(return_value="[RAW] checkpoint")
 
         await consolidator.maybe_consolidate_by_tokens(session, runtime=runtime)
 
@@ -613,27 +653,62 @@ class TestCompactIdleSession:
         assert reloaded.last_archived == 2
         assert [message["content"] for message in reloaded.get_history()] == ["hello", "hi"]
 
+    @pytest.mark.asyncio
+    async def test_idle_compaction_with_no_new_messages_is_noop(
+        self, real_consolidator, mock_provider, store, runtime
+    ):
+        sessions = real_consolidator.sessions
+        session = sessions.get_or_create("cli:archived-idle")
+        session.add_message("user", "already archived")
+        session.add_message("assistant", "old answer")
+        session.last_archived = 2
+        sessions.save(session)
+        sessions.invalidate("cli:archived-idle")
+
+        result = await real_consolidator.compact_idle_session(
+            "cli:archived-idle",
+            runtime=runtime,
+        )
+
+        assert result == ""
+        mock_provider.chat_with_retry.assert_not_awaited()
+        reloaded = sessions.get_or_create("cli:archived-idle")
+        assert reloaded.last_archived == 2
+        assert "_last_summary" not in reloaded.metadata
+        assert store.read_unprocessed_history(since_cursor=0) == []
+
     @pytest.mark.asyncio
     async def test_new_messages_advance_existing_archive_progress(
         self, real_consolidator, mock_provider, runtime
     ):
-        mock_provider.chat_with_retry.return_value = MagicMock(
-            content="Summary.", finish_reason="stop"
-        )
+        mock_provider.chat_with_retry.side_effect = [
+            MagicMock(content="First replacement checkpoint.", finish_reason="stop"),
+            MagicMock(content="Second replacement checkpoint.", finish_reason="stop"),
+        ]
         sessions = real_consolidator.sessions
         session = sessions.get_or_create("cli:incremental")
         session.add_message("user", "first user")
         session.add_message("assistant", "first assistant")
         sessions.save(session)
 
-        await real_consolidator.compact_idle_session("cli:incremental", runtime=runtime)
+        first = await real_consolidator.compact_idle_session(
+            "cli:incremental",
+            runtime=runtime,
+        )
         current = sessions.get_or_create("cli:incremental")
         current.add_message("user", "second user")
         current.add_message("assistant", "second assistant")
         sessions.save(current)
-        await real_consolidator.compact_idle_session("cli:incremental", runtime=runtime)
+        second = await real_consolidator.compact_idle_session(
+            "cli:incremental",
+            runtime=runtime,
+        )
 
+        assert first == "First replacement checkpoint."
+        assert second == "Second replacement checkpoint."
         assert mock_provider.chat_with_retry.await_count == 2
+        latest_build = real_consolidator.archiver._build_messages.call_args_list[-1].kwargs
+        assert latest_build["session_summary"]["text"] == "First replacement checkpoint."
         latest_messages = mock_provider.chat_with_retry.await_args_list[-1].kwargs["messages"]
         assert [message["content"] for message in latest_messages[1:5]] == [
             "first user",
@@ -641,8 +716,91 @@ class TestCompactIdleSession:
             "second user",
             "second assistant",
         ]
-        assert "final 2 conversation messages" in latest_messages[-1]["content"]
-        assert sessions.get_or_create("cli:incremental").last_archived == 4
+        assert latest_messages[-1]["content"] == _ARCHIVE_PROMPT
+        sessions.invalidate("cli:incremental")
+        reloaded = sessions.get_or_create("cli:incremental")
+        assert reloaded.last_archived == 4
+        assert reloaded.metadata["_last_summary"]["text"] == second
+
+    @pytest.mark.asyncio
+    async def test_raw_fallback_preserves_previous_checkpoint_and_new_chunk(
+        self,
+        real_consolidator,
+        mock_provider,
+        store,
+        runtime,
+    ):
+        mock_provider.chat_with_retry.side_effect = [
+            LLMResponse(content="Earlier durable checkpoint.", finish_reason="stop"),
+            RuntimeError("LLM unavailable"),
+        ]
+        sessions = real_consolidator.sessions
+        session = sessions.get_or_create("cli:cumulative-fallback")
+        session.add_message("user", "first user")
+        session.add_message("assistant", "first answer")
+        sessions.save(session)
+
+        await real_consolidator.compact_idle_session(
+            "cli:cumulative-fallback",
+            runtime=runtime,
+        )
+        current = sessions.get_or_create("cli:cumulative-fallback")
+        current.add_message("user", "second user")
+        current.add_message("assistant", "newest working state")
+        sessions.save(current)
+
+        fallback = await real_consolidator.compact_idle_session(
+            "cli:cumulative-fallback",
+            runtime=runtime,
+        )
+
+        assert fallback is not None
+        assert "[Previous archived context]" in fallback
+        assert "Earlier durable checkpoint." in fallback
+        assert "[Newly archived raw context]" in fallback
+        assert "newest working state" in fallback
+        entries = store.read_unprocessed_history(0)
+        assert entries[0]["content"] == "Earlier durable checkpoint."
+        assert entries[1]["content"].startswith("[RAW] 2 messages")
+        sessions.invalidate("cli:cumulative-fallback")
+        reloaded = sessions.get_or_create("cli:cumulative-fallback")
+        assert reloaded.metadata["_last_summary"]["text"] == fallback
+
+    @pytest.mark.asyncio
+    async def test_nothing_keeps_previous_replacement_checkpoint(
+        self,
+        real_consolidator,
+        mock_provider,
+        runtime,
+    ):
+        mock_provider.chat_with_retry.side_effect = [
+            LLMResponse(content="Existing checkpoint.", finish_reason="stop"),
+            LLMResponse(content="(nothing)", finish_reason="stop"),
+        ]
+        sessions = real_consolidator.sessions
+        session = sessions.get_or_create("cli:nothing-after-summary")
+        session.add_message("user", "important first turn")
+        session.add_message("assistant", "important result")
+        sessions.save(session)
+        await real_consolidator.compact_idle_session(
+            "cli:nothing-after-summary",
+            runtime=runtime,
+        )
+
+        current = sessions.get_or_create("cli:nothing-after-summary")
+        current.add_message("user", "thanks")
+        current.add_message("assistant", "you're welcome")
+        sessions.save(current)
+        result = await real_consolidator.compact_idle_session(
+            "cli:nothing-after-summary",
+            runtime=runtime,
+        )
+
+        assert result == "(nothing)"
+        sessions.invalidate("cli:nothing-after-summary")
+        reloaded = sessions.get_or_create("cli:nothing-after-summary")
+        assert reloaded.last_archived == 4
+        assert reloaded.metadata["_last_summary"]["text"] == "Existing checkpoint."
 
     @pytest.mark.asyncio
     async def test_concurrent_append_remains_unarchived(
@@ -792,11 +950,16 @@ class TestCompactIdleSession:
         result = await real_consolidator.compact_idle_session(
             "cli:nothing", runtime=runtime, max_suffix=4
         )
+        second = await real_consolidator.compact_idle_session(
+            "cli:nothing", runtime=runtime, max_suffix=4
+        )
         assert result == "(nothing)"
+        assert second == ""
 
         reloaded = sessions.get_or_create("cli:nothing")
         assert "_last_summary" not in reloaded.metadata
         assert real_consolidator.store.read_unprocessed_history(0) == []
+        mock_provider.chat_with_retry.assert_awaited_once()
 
     @pytest.mark.asyncio
     async def test_llm_failure_preserves_history_but_advances_replay_boundary(
@@ -813,7 +976,8 @@ class TestCompactIdleSession:
         result = await real_consolidator.compact_idle_session(
             "cli:fail", runtime=runtime, max_suffix=4
         )
-        assert result is None
+        assert result is not None
+        assert "[RAW]" in result
 
         # raw_archive should have been called (history.jsonl gets an entry)
         entries = store.read_unprocessed_history(since_cursor=0)
@@ -823,6 +987,7 @@ class TestCompactIdleSession:
         assert len(reloaded.messages) == 20
         assert reloaded.messages[0]["content"] == "u0"
         assert reloaded.last_archived == 20
+        assert reloaded.metadata["_last_summary"]["text"] == result
         assert [m["content"] for m in reloaded.get_history(max_messages=20)] == [
             "u6",
             "a6",
@@ -863,11 +1028,10 @@ class TestCompactIdleSession:
         archived_call = mock_provider.chat_with_retry.call_args
         sent_messages = archived_call.kwargs["messages"]
         sent_content = [message.get("content") for message in sent_messages]
-        # The ordinary replay prefix contributes recent context, while the
-        # temporary instruction limits the new overview to the unarchived tail.
+        # The replacement overview covers all model-visible conversation context.
         assert "u0" not in sent_content
         assert "u26" in sent_content
-        assert "final 10 conversation messages" in sent_messages[-1]["content"]
+        assert sent_messages[-1]["content"] == _ARCHIVE_PROMPT
 
     @pytest.mark.asyncio
     async def test_full_archive_keeps_extended_legal_replay_suffix(
@@ -956,9 +1120,9 @@ class TestCompactIdleSession:
             "user",
         ]
         assert sent_messages[2]["tool_calls"][0]["id"] == "call-1"
-        assert "final 4 conversation messages" in sent_messages[-1]["content"]
+        assert sent_messages[-1]["content"] == _ARCHIVE_PROMPT
         assert call["tools"] == tools
-        assert call["tool_choice"] == "none"
+        assert "tool_choice" not in call
 
         reloaded = sessions.get_or_create("cli:tool-history")
         assert len(reloaded.messages) == 4
@@ -996,7 +1160,8 @@ class TestCompactIdleSession:
             runtime=runtime,
         )
 
-        assert result is None
+        assert result is not None
+        assert "[RAW]" in result
         entries = store.read_unprocessed_history(since_cursor=0)
         assert len(entries) == 1
         assert entries[0]["content"].startswith("[RAW] ")
@@ -1026,7 +1191,8 @@ class TestCompactIdleSession:
             runtime=runtime,
         )
 
-        assert result is None
+        assert result is not None
+        assert "[RAW]" in result
         entries = store.read_unprocessed_history(since_cursor=0)
         assert len(entries) == 1
         assert entries[0]["content"].startswith("[RAW] ")
@@ -1052,7 +1218,8 @@ class TestCompactIdleSession:
             runtime=runtime,
         )
 
-        assert result is None
+        assert result is not None
+        assert "[RAW]" in result
         mock_provider.chat_with_retry.assert_not_awaited()
         entries = store.read_unprocessed_history(since_cursor=0)
         assert len(entries) == 1
@@ -1060,7 +1227,7 @@ class TestCompactIdleSession:
         assert sessions.get_or_create("sdk:oversized").last_archived == 1
 
     @pytest.mark.asyncio
-    async def test_incremental_scope_counts_only_model_visible_messages(
+    async def test_archive_context_contains_only_model_visible_messages(
         self,
         real_consolidator,
         mock_provider,
@@ -1093,7 +1260,7 @@ class TestCompactIdleSession:
             "new user",
             "new answer",
         ]
-        assert "final 2 conversation messages" in sent[-1]["content"]
+        assert sent[-1]["content"] == _ARCHIVE_PROMPT
 
     @pytest.mark.asyncio
     async def test_reuses_real_prefix_for_unified_session_workspace(
@@ -1126,8 +1293,6 @@ class TestCompactIdleSession:
             current_message="next project question",
             channel="websocket",
             workspace=project,
-            session_key=session.key,
-            unified_session=True,
         )
 
         await loop.consolidator.compact_idle_session(
@@ -1137,7 +1302,7 @@ class TestCompactIdleSession:
 
         sent_messages = runtime.provider.chat_with_retry.call_args.kwargs["messages"]
         assert sent_messages[:-1] == ordinary_messages[:-1]
-        assert "final 2 conversation messages" in sent_messages[-1]["content"]
+        assert sent_messages[-1]["content"] == _ARCHIVE_PROMPT
         system = sent_messages[0]["content"]
         assert "PROJECT_WORKSPACE_MARKER" in system
         assert "GLOBAL_WORKSPACE_MARKER" not in system
@@ -1307,6 +1472,21 @@ class TestRawArchiveTruncation:
         assert len(entries) == 1
         assert "hello" in entries[0]["content"]
 
+    def test_raw_archive_returns_the_sanitized_persisted_checkpoint(self, store):
+        messages = [
+            {
+                "role": "user",
+                "content": "<think>PRIVATE_REASONING</think>visible result",
+            }
+        ]
+
+        checkpoint = store.raw_archive(messages, session_key="cli:test")
+
+        persisted = store.read_unprocessed_history(since_cursor=0)[0]["content"]
+        assert checkpoint == persisted
+        assert "PRIVATE_REASONING" not in checkpoint
+        assert "visible result" in checkpoint
+
     def test_raw_archive_excludes_model_only_runtime_context(self, store):
         content, marker = append_runtime_context(
             "ship the feature",
@@ -1338,21 +1518,40 @@ class TestRawArchiveTruncation:
 
 
 class TestArchivePersistence:
-    async def test_oversized_summary_is_capped_before_append(
+    async def test_archive_returns_the_sanitized_persisted_summary(
+        self, consolidator, mock_provider, store, runtime
+    ):
+        mock_provider.chat_with_retry.return_value = MagicMock(
+            content="<think>PRIVATE_REASONING</think>safe summary",
+            finish_reason="stop",
+            has_tool_calls=False,
+        )
+
+        summary = await _archive(
+            consolidator,
+            [{"role": "user", "content": "hi"}],
+            runtime,
+        )
+
+        persisted = store.read_unprocessed_history(since_cursor=0)[0]["content"]
+        assert summary == persisted == "safe summary"
+
+    async def test_oversized_summary_uses_history_emergency_cap(
         self, consolidator, mock_provider, store, runtime
     ):
         """A pathologically large LLM summary must not land full-length in
         history.jsonl — that would re-open the #3412 bloat vector from the
         *success* path instead of the fallback path."""
         mock_provider.chat_with_retry.return_value = MagicMock(
-            content="S" * (_ARCHIVE_SUMMARY_MAX_CHARS * 10),
+            content="S" * (_HISTORY_ENTRY_HARD_CAP * 2),
             finish_reason="stop",
         )
-        await _archive(
+        summary = await _archive(
             consolidator,
             [{"role": "user", "content": "hi"}],
             runtime,
         )
 
         entry = store.read_unprocessed_history(since_cursor=0)[0]
-        assert len(entry["content"]) <= _ARCHIVE_SUMMARY_MAX_CHARS + 50
+        assert len(entry["content"]) <= _HISTORY_ENTRY_HARD_CAP + 50
+        assert summary == entry["content"]