feat: preserve Responses reasoning state and compact context (#5172)

tests/agent/test_runner_core.py

@@ -11,7 +11,13 @@ import pytest
 
 from agent.runner_helpers import make_run_spec
 from nanobot.config.schema import AgentDefaults
-from nanobot.providers.base import LLMProvider, LLMResponse, ToolCallRequest
+from nanobot.providers.base import (
+    LLMProvider,
+    LLMResponse,
+    ProviderCallContext,
+    ProviderConversationState,
+    ToolCallRequest,
+)
 
 _MAX_TOOL_RESULT_CHARS = AgentDefaults().max_tool_result_chars
 
@@ -73,6 +79,311 @@ async def test_runner_preserves_reasoning_fields_and_tool_results():
     )
 
 
+@pytest.mark.asyncio
+async def test_runner_replays_provider_state_without_chat_projection_duplicates():
+    from nanobot.agent.runner import AgentRunner
+
+    provider = MagicMock(spec=LLMProvider)
+    provider.can_resume_conversation_state.return_value = True
+    provider.supports_native_compaction.return_value = False
+    captured_second_kwargs: dict = {}
+    checkpoints: list[dict] = []
+    calls = 0
+
+    async def checkpoint(payload: dict) -> None:
+        checkpoints.append(payload)
+
+    first_state = ProviderConversationState(
+        kind="openai_responses",
+        provider="openai:test",
+        model="gpt-5.6",
+        version=1,
+        payload={"items": [{"type": "reasoning", "encrypted_content": "opaque"}]},
+    )
+    second_state = ProviderConversationState(
+        kind="openai_responses",
+        provider="openai:test",
+        model="gpt-5.6",
+        version=1,
+        payload={"items": [{"type": "message", "role": "assistant"}]},
+    )
+
+    async def chat_with_retry(**kwargs):
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            provider_context = kwargs["provider_context"]
+            assert isinstance(provider_context, ProviderCallContext)
+            assert provider_context.conversation_state is None
+            return LLMResponse(
+                content=None,
+                tool_calls=[
+                    ToolCallRequest(
+                        id="call_1|fc_1",
+                        name="list_dir",
+                        arguments={"path": "."},
+                    ),
+                ],
+                provider_state=first_state,
+            )
+        captured_second_kwargs.update(kwargs)
+        return LLMResponse(content="done", provider_state=second_state)
+
+    provider.chat_with_retry = chat_with_retry
+    tools = MagicMock()
+    tools.get_definitions.return_value = []
+    tools.execute = AsyncMock(return_value="tool result")
+
+    result = await AgentRunner().run(make_run_spec(
+        provider,
+        initial_messages=[
+            {"role": "system", "content": "system"},
+            {"role": "user", "content": "do task"},
+        ],
+        tools=tools,
+        model="gpt-5.6",
+        max_iterations=3,
+        max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
+        checkpoint_callback=checkpoint,
+    ))
+
+    provider_context = captured_second_kwargs["provider_context"]
+    assert isinstance(provider_context, ProviderCallContext)
+    assert provider_context.conversation_state is not None
+    assert provider_context.conversation_state.payload == first_state.payload
+    assert provider_context.conversation_state.pending_messages == [{
+        "role": "tool",
+        "tool_call_id": "call_1|fc_1",
+        "name": "list_dir",
+        "content": "tool result",
+    }]
+    assert not any(
+        message.get("role") == "assistant"
+        for message in provider_context.conversation_state.pending_messages
+    )
+    assert result.provider_state is not None
+    assert result.provider_state.payload == second_state.payload
+    assert result.provider_state.pending_messages == []
+    assert checkpoints[0]["phase"] == "awaiting_tools"
+    assert "provider_state" not in checkpoints[0]
+    assert checkpoints[1]["phase"] == "tools_completed"
+    assert checkpoints[1]["provider_state"].pending_messages == [{
+        "role": "tool",
+        "tool_call_id": "call_1|fc_1",
+        "name": "list_dir",
+        "content": "tool result",
+    }]
+    assert checkpoints[2]["phase"] == "final_response"
+    assert checkpoints[2]["provider_state"].payload == second_state.payload
+
+
+@pytest.mark.asyncio
+async def test_runner_governs_tool_result_before_adding_it_to_provider_state():
+    from nanobot.agent.runner import AgentRunner
+
+    provider = MagicMock(spec=LLMProvider)
+    provider.can_resume_conversation_state.return_value = True
+    provider.supports_native_compaction.return_value = False
+    calls = 0
+    captured_context: ProviderCallContext | None = None
+    checkpoints: list[dict] = []
+    state = ProviderConversationState(
+        kind="openai_responses",
+        provider="openai:test",
+        model="gpt-5.6",
+        version=1,
+        payload={"items": [{"type": "reasoning", "encrypted_content": "opaque"}]},
+    )
+
+    async def chat_with_retry(**kwargs):
+        nonlocal calls, captured_context
+        calls += 1
+        if calls == 1:
+            return LLMResponse(
+                content=None,
+                tool_calls=[
+                    ToolCallRequest(
+                        id="call_1",
+                        name="read_file",
+                        arguments={"path": "large.txt"},
+                    ),
+                ],
+                provider_state=state,
+            )
+        captured_context = kwargs["provider_context"]
+        return LLMResponse(content="done")
+
+    provider.chat_with_retry = chat_with_retry
+    tools = MagicMock()
+    tools.get_definitions.return_value = []
+    tools.execute = AsyncMock(return_value="x" * 5_000)
+
+    async def checkpoint(payload: dict) -> None:
+        checkpoints.append(payload)
+
+    await AgentRunner().run(make_run_spec(
+        provider,
+        initial_messages=[
+            {"role": "system", "content": "system"},
+            {"role": "user", "content": "read the file"},
+        ],
+        tools=tools,
+        model="gpt-5.6",
+        context_window_tokens=3_000,
+        context_block_limit=200,
+        max_tokens=1_000,
+        max_iterations=3,
+        max_tool_result_chars=10_000,
+        checkpoint_callback=checkpoint,
+    ))
+
+    assert captured_context is not None
+    assert captured_context.conversation_state is not None
+    pending = captured_context.conversation_state.pending_messages
+    assert len(pending) == 1
+    assert pending[0]["role"] == "tool"
+    assert "compacted to fit context" in pending[0]["content"]
+    assert pending[0]["content"] != "x" * 5_000
+    completed_checkpoint = next(
+        checkpoint
+        for checkpoint in checkpoints
+        if checkpoint["phase"] == "tools_completed"
+    )
+    checkpoint_pending = completed_checkpoint["provider_state"].pending_messages
+    assert "compacted to fit context" in checkpoint_pending[0]["content"]
+    assert checkpoint_pending[0]["content"] != "x" * 5_000
+
+
+@pytest.mark.asyncio
+async def test_injected_final_response_checkpoint_includes_provider_state():
+    from nanobot.agent.runner import AgentRunner
+
+    provider = MagicMock(spec=LLMProvider)
+    provider.can_resume_conversation_state.return_value = True
+    provider.supports_native_compaction.return_value = False
+    first_state = ProviderConversationState(
+        kind="openai_responses",
+        provider="openai:test",
+        model="gpt-5.6",
+        version=1,
+        payload={"items": [{"type": "message", "content": "first answer"}]},
+    )
+    second_state = ProviderConversationState(
+        kind="openai_responses",
+        provider="openai:test",
+        model="gpt-5.6",
+        version=1,
+        payload={"items": [{"type": "message", "content": "second answer"}]},
+    )
+    provider.chat_with_retry = AsyncMock(side_effect=[
+        LLMResponse(content="first answer", provider_state=first_state),
+        LLMResponse(content="second answer", provider_state=second_state),
+    ])
+    tools = MagicMock()
+    tools.get_definitions.return_value = []
+    checkpoints: list[dict] = []
+    injections = [[{"role": "user", "content": "follow up"}], []]
+
+    async def checkpoint(payload: dict) -> None:
+        checkpoints.append(payload)
+
+    async def inject() -> list[dict]:
+        return injections.pop(0)
+
+    await AgentRunner().run(make_run_spec(
+        provider,
+        initial_messages=[{"role": "user", "content": "start"}],
+        tools=tools,
+        model="gpt-5.6",
+        max_iterations=3,
+        max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
+        checkpoint_callback=checkpoint,
+        injection_callback=inject,
+    ))
+
+    assert checkpoints[0]["phase"] == "final_response"
+    assert checkpoints[0]["provider_state"].payload == first_state.payload
+
+
+@pytest.mark.asyncio
+async def test_runner_preserves_last_completed_provider_state_on_model_error():
+    from nanobot.agent.runner import AgentRunner
+
+    provider = MagicMock(spec=LLMProvider)
+    provider.can_resume_conversation_state.return_value = True
+    provider.chat_with_retry = AsyncMock(return_value=LLMResponse(
+        content="temporary upstream failure",
+        finish_reason="error",
+        error_kind="timeout",
+    ))
+    tools = MagicMock()
+    tools.get_definitions.return_value = []
+    state = ProviderConversationState(
+        kind="openai_responses",
+        provider="openai:test",
+        model="gpt-5.6",
+        version=1,
+        payload={"items": [{"type": "reasoning", "encrypted_content": "opaque"}]},
+    )
+    unsaved_input = {"role": "user", "content": "ephemeral follow-up"}
+
+    result = await AgentRunner().run(make_run_spec(
+        provider,
+        initial_messages=[
+            {"role": "system", "content": "system"},
+            unsaved_input,
+        ],
+        tools=tools,
+        model="gpt-5.6",
+        max_iterations=1,
+        max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
+        provider_state=state.with_pending_messages([unsaved_input]),
+    ))
+
+    assert result.stop_reason == "error"
+    assert result.provider_state is not None
+    assert result.provider_state.payload == state.payload
+    assert result.provider_state.pending_messages[0] == unsaved_input
+    assert result.provider_state.pending_messages[1]["role"] == "assistant"
+    assert "model error" in result.provider_state.pending_messages[1]["content"]
+
+
+@pytest.mark.asyncio
+async def test_runner_discards_provider_state_on_non_retryable_model_error():
+    from nanobot.agent.runner import AgentRunner
+
+    provider = MagicMock(spec=LLMProvider)
+    provider.can_resume_conversation_state.return_value = True
+    provider.chat_with_retry = AsyncMock(return_value=LLMResponse(
+        content="context length exceeded",
+        finish_reason="error",
+        error_status_code=400,
+        error_should_retry=False,
+    ))
+    tools = MagicMock()
+    tools.get_definitions.return_value = []
+    state = ProviderConversationState(
+        kind="openai_responses",
+        provider="openai:test",
+        model="gpt-5.6",
+        version=1,
+        payload={"items": [{"type": "reasoning", "encrypted_content": "opaque"}]},
+    )
+
+    result = await AgentRunner().run(make_run_spec(
+        provider,
+        initial_messages=[{"role": "user", "content": "continue"}],
+        tools=tools,
+        model="gpt-5.6",
+        max_iterations=1,
+        max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
+        provider_state=state,
+    ))
+
+    assert result.stop_reason == "error"
+    assert result.provider_state is None
+
+
 @pytest.mark.asyncio
 async def test_runner_returns_max_iterations_fallback():
     from nanobot.agent.runner import AgentRunner
@@ -422,6 +733,66 @@ async def test_runner_retries_empty_final_response_with_summary_prompt():
     assert result.usage["completion_tokens"] == 9
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("finish_reason", ["refusal", "content_filter"])
+async def test_runner_does_not_retry_blank_policy_terminal(
+    finish_reason: str,
+) -> None:
+    from nanobot.agent.runner import AgentRunner
+    from nanobot.utils.runtime import EMPTY_FINAL_RESPONSE_MESSAGE
+
+    provider = MagicMock(spec=LLMProvider)
+    provider.chat_with_retry = AsyncMock(return_value=LLMResponse(
+        content=None,
+        finish_reason=finish_reason,
+    ))
+    tools = MagicMock()
+    tools.get_definitions.return_value = []
+
+    result = await AgentRunner().run(make_run_spec(
+        provider,
+        initial_messages=[{"role": "user", "content": "do task"}],
+        tools=tools,
+        model="test-model",
+        max_iterations=3,
+        max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
+    ))
+
+    assert provider.chat_with_retry.await_count == 1
+    assert result.final_content == EMPTY_FINAL_RESPONSE_MESSAGE
+    assert result.stop_reason == "empty_final_response"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("finish_reason", ["refusal", "content_filter"])
+async def test_runner_does_not_auto_continue_goal_after_policy_terminal(
+    finish_reason: str,
+) -> None:
+    from nanobot.agent.runner import AgentRunner
+
+    provider = MagicMock(spec=LLMProvider)
+    provider.chat_with_retry = AsyncMock(return_value=LLMResponse(
+        content="Request blocked by provider policy.",
+        finish_reason=finish_reason,
+    ))
+    tools = MagicMock()
+    tools.get_definitions.return_value = []
+
+    result = await AgentRunner().run(make_run_spec(
+        provider,
+        initial_messages=[{"role": "user", "content": "do task"}],
+        tools=tools,
+        model="test-model",
+        max_iterations=3,
+        max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
+        goal_active_predicate=lambda: True,
+    ))
+
+    assert provider.chat_with_retry.await_count == 1
+    assert result.final_content == "Request blocked by provider policy."
+    assert result.stop_reason == "completed"
+
+
 @pytest.mark.asyncio
 async def test_runner_uses_specific_message_after_empty_finalization_retry():
     """After silent retries + finalization all return empty, stop_reason is empty_final_response."""
@@ -450,6 +821,56 @@ async def test_runner_uses_specific_message_after_empty_finalization_retry():
     assert result.stop_reason == "empty_final_response"
 
 
+@pytest.mark.asyncio
+async def test_empty_finalization_retry_discards_candidate_provider_state():
+    from nanobot.agent.runner import AgentRunner
+
+    candidate = ProviderConversationState(
+        kind="openai_responses",
+        provider="openai:test",
+        model="test-model",
+        version=1,
+        payload={
+            "items": [{
+                "type": "function_call",
+                "call_id": "call_1",
+                "name": "exec",
+                "arguments": "{}",
+            }],
+        },
+    )
+    provider = MagicMock(spec=LLMProvider)
+    provider.can_resume_conversation_state.return_value = True
+    provider.chat_with_retry = AsyncMock(side_effect=[
+        LLMResponse(content=None, tool_calls=[], usage={}),
+        LLMResponse(content=None, tool_calls=[], usage={}),
+        LLMResponse(
+            content="finalized without tools",
+            tool_calls=[ToolCallRequest(id="call_1", name="exec", arguments={})],
+            finish_reason="stop",
+            provider_state=candidate,
+            usage={},
+        ),
+    ])
+    tools = MagicMock()
+    tools.get_definitions.return_value = []
+    tools.execute = AsyncMock(return_value="must not run")
+
+    runner = AgentRunner()
+    result = await runner.run(make_run_spec(
+        provider,
+        initial_messages=[{"role": "user", "content": "do task"}],
+        tools=tools,
+        model="test-model",
+        max_iterations=3,
+        max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
+    ))
+
+    tools.execute.assert_not_awaited()
+    assert result.final_content == "finalized without tools"
+    assert result.provider_state is None
+
+
 @pytest.mark.asyncio
 async def test_runner_length_recovery_returns_all_segments():
     """Recovered output segments are returned together instead of only the tail."""