From 81a5af23522beb45327417245917473804e50be5 Mon Sep 17 00:00:00 2001 From: Xubin Ren Date: Thu, 23 Apr 2026 19:55:30 +0000 Subject: [PATCH] test(consolidation): add regression tests for tiktoken truncation path and history char cap Cover two untested boundaries from #3412: - _truncate_to_token_budget with positive budget exercises tiktoken - _MAX_HISTORY_CHARS caps Recent History section in system prompt Made-with: Cursor --- tests/agent/test_consolidator.py | 17 +++++++++++++++++ tests/agent/test_context_prompt_cache.py | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/tests/agent/test_consolidator.py b/tests/agent/test_consolidator.py index 77aee609d..58dce3c64 100644 --- a/tests/agent/test_consolidator.py +++ b/tests/agent/test_consolidator.py @@ -230,3 +230,20 @@ class TestArchiveTruncation: # budget = 500 - 100 - 1024 = negative, fallback char-based # Should be truncated assert len(user_content) < 250_000 + + async def test_archive_truncates_via_tiktoken_with_positive_budget(self, consolidator, mock_provider, store): + """Positive token budget should use tiktoken for precise truncation.""" + consolidator.context_window_tokens = 10_000 + consolidator._SAFETY_BUFFER = 0 + # budget = 10000 - 100 - 0 = 9900 tokens + big_messages = [{"role": "user", "content": "word " * 50_000}] + mock_provider.chat_with_retry.return_value = MagicMock( + content="Summary.", finish_reason="stop" + ) + await consolidator.archive(big_messages) + + import tiktoken + enc = tiktoken.get_encoding("cl100k_base") + sent_content = mock_provider.chat_with_retry.call_args.kwargs["messages"][1]["content"] + token_count = len(enc.encode(sent_content)) + assert token_count <= 9_900 + 10 # small margin for truncation suffix diff --git a/tests/agent/test_context_prompt_cache.py b/tests/agent/test_context_prompt_cache.py index ad132e837..ea1052ca0 100644 --- a/tests/agent/test_context_prompt_cache.py +++ b/tests/agent/test_context_prompt_cache.py @@ -116,6 +116,20 @@ def test_recent_history_capped_at_max(tmp_path) -> None: assert f"entry-{builder._MAX_RECENT_HISTORY + 19}" in prompt +def test_recent_history_truncated_at_max_chars(tmp_path) -> None: + """Recent History section must be truncated at _MAX_HISTORY_CHARS.""" + workspace = _make_workspace(tmp_path) + builder = ContextBuilder(workspace) + + big_entry = "x" * (builder._MAX_HISTORY_CHARS + 5_000) + builder.memory.append_history(big_entry) + + prompt = builder.build_system_prompt() + history_section = prompt.split("# Recent History\n\n", 1) + assert len(history_section) == 2 + assert len(history_section[1]) < builder._MAX_HISTORY_CHARS + 200 + + def test_no_recent_history_when_dream_has_processed_all(tmp_path) -> None: """If Dream has consumed everything, no Recent History section should appear.""" workspace = _make_workspace(tmp_path)