mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-08-07 21:08:34 +03:00
feat: preserve Responses reasoning state and compact context (#5172)
This commit is contained in:
chengyongru
@@ -11,7 +11,7 @@ from nanobot.providers.azure_openai_provider import (
|
||||
AzureOpenAIProvider,
|
||||
_AzureTokenProvider,
|
||||
)
|
||||
from nanobot.providers.base import LLMResponse
|
||||
from nanobot.providers.base import LLMResponse, ProviderCallContext
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Init & validation
|
||||
@@ -234,6 +234,7 @@ def test_build_body_basic():
|
||||
assert body["max_output_tokens"] == 4096
|
||||
assert body["store"] is False
|
||||
assert "reasoning" not in body
|
||||
assert "include" not in body
|
||||
# input should contain the converted user message only (system extracted)
|
||||
assert any(
|
||||
item.get("role") == "user"
|
||||
@@ -241,6 +242,30 @@ def test_build_body_basic():
|
||||
)
|
||||
|
||||
|
||||
def test_build_body_enables_server_compaction():
|
||||
provider = AzureOpenAIProvider(
|
||||
api_key="k",
|
||||
api_base="https://res.openai.azure.com",
|
||||
default_model="gpt-5.6",
|
||||
)
|
||||
|
||||
body = provider._build_body(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
None,
|
||||
None,
|
||||
10_000,
|
||||
0.1,
|
||||
"high",
|
||||
None,
|
||||
provider_context=ProviderCallContext(context_window_tokens=200_000),
|
||||
)
|
||||
|
||||
assert body["context_management"] == [{
|
||||
"type": "compaction",
|
||||
"compact_threshold": 180_000,
|
||||
}]
|
||||
|
||||
|
||||
def test_build_body_max_tokens_minimum():
|
||||
"""max_output_tokens should never be less than 1."""
|
||||
provider = AzureOpenAIProvider(api_key="k", api_base="https://r.com", default_model="gpt-4o")
|
||||
@@ -358,6 +383,38 @@ async def test_chat_success():
|
||||
assert result.usage["prompt_tokens"] == 10
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_retries_without_unsupported_server_compaction():
|
||||
provider = AzureOpenAIProvider(
|
||||
api_key="test-key",
|
||||
api_base="https://test.openai.azure.com",
|
||||
default_model="gpt-5.6",
|
||||
)
|
||||
|
||||
class UnsupportedCompactionError(Exception):
|
||||
status_code = 400
|
||||
body = {"error": {"message": "Unknown parameter: context_management"}}
|
||||
|
||||
provider._client.responses = MagicMock()
|
||||
provider._client.responses.create = AsyncMock(side_effect=[
|
||||
UnsupportedCompactionError(),
|
||||
_make_sdk_response(content="compaction fallback"),
|
||||
])
|
||||
|
||||
result = await provider.chat(
|
||||
[{"role": "user", "content": "Hi"}],
|
||||
provider_context=ProviderCallContext(context_window_tokens=200_000),
|
||||
)
|
||||
|
||||
create = provider._client.responses.create
|
||||
assert result.content == "compaction fallback"
|
||||
assert result.provider_state is not None
|
||||
assert create.await_count == 2
|
||||
assert "context_management" in create.call_args_list[0].kwargs
|
||||
assert "context_management" not in create.call_args_list[1].kwargs
|
||||
assert provider.supports_native_compaction() is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_uses_default_model():
|
||||
provider = AzureOpenAIProvider(
|
||||
@@ -411,6 +468,7 @@ async def test_chat_with_tool_calls():
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].name == "get_weather"
|
||||
assert result.tool_calls[0].arguments == {"location": "SF"}
|
||||
assert result.provider_state is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -510,6 +568,7 @@ async def test_chat_stream_with_tool_calls():
|
||||
item_done.name = "get_weather"
|
||||
ev_item_done = MagicMock(type="response.output_item.done", item=item_done)
|
||||
resp_obj = MagicMock(status="completed")
|
||||
resp_obj.model_dump.return_value = {"status": "completed", "output": []}
|
||||
ev_completed = MagicMock(type="response.completed", response=resp_obj)
|
||||
|
||||
async def mock_stream():
|
||||
@@ -527,6 +586,7 @@ async def test_chat_stream_with_tool_calls():
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].name == "get_weather"
|
||||
assert result.tool_calls[0].arguments == {"location": "SF"}
|
||||
assert result.provider_state is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Tests for provider-owned conversation-state lifecycle coordination."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nanobot.providers.base import (
|
||||
LLMProvider,
|
||||
LLMResponse,
|
||||
ProviderConversationState,
|
||||
ToolCallRequest,
|
||||
)
|
||||
from nanobot.providers.conversation_state import (
|
||||
ProviderConversationStateController,
|
||||
allows_conversation_message_merge,
|
||||
)
|
||||
|
||||
|
||||
def _provider(*, resumable: bool = True, compact: bool = False) -> MagicMock:
|
||||
provider = MagicMock(spec=LLMProvider)
|
||||
provider.can_resume_conversation_state.return_value = resumable
|
||||
provider.supports_native_compaction.return_value = compact
|
||||
return provider
|
||||
|
||||
|
||||
def _state(label: str, *, pending: list[dict] | None = None) -> ProviderConversationState:
|
||||
return ProviderConversationState(
|
||||
kind="openai_responses",
|
||||
provider="openai:test",
|
||||
model="gpt-5.6",
|
||||
version=1,
|
||||
payload={"items": [{"type": "reasoning", "encrypted_content": label}]},
|
||||
pending_messages=pending or [],
|
||||
)
|
||||
|
||||
|
||||
def test_controller_replays_only_messages_after_provider_output() -> None:
|
||||
provider = _provider()
|
||||
messages = [
|
||||
{"role": "system", "content": "system"},
|
||||
{"role": "user", "content": "run a tool"},
|
||||
]
|
||||
controller = ProviderConversationStateController(
|
||||
provider=provider,
|
||||
model="gpt-5.6",
|
||||
messages=messages,
|
||||
)
|
||||
state = _state("first")
|
||||
|
||||
controller.prepare_request(messages, context_window_tokens=200_000)
|
||||
response = LLMResponse(content=None, provider_state=state)
|
||||
controller.observe_response(response, messages)
|
||||
assert allows_conversation_message_merge(messages[-1]) is False
|
||||
|
||||
messages.append(controller.project_response_message(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function"}],
|
||||
},
|
||||
response,
|
||||
))
|
||||
tool_message = {
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": "tool result",
|
||||
}
|
||||
messages.append(tool_message)
|
||||
|
||||
provider_context = controller.prepare_request(
|
||||
messages,
|
||||
context_window_tokens=200_000,
|
||||
)
|
||||
|
||||
assert provider_context is not None
|
||||
assert provider_context.conversation_state is not None
|
||||
assert provider_context.conversation_state.payload == state.payload
|
||||
assert provider_context.conversation_state.pending_messages == [tool_message]
|
||||
assert controller.checkpoint(messages).pending_messages == [tool_message]
|
||||
|
||||
|
||||
def test_controller_uses_governed_messages_for_provider_state_delta() -> None:
|
||||
provider = _provider()
|
||||
messages = [
|
||||
{"role": "user", "content": "run a tool"},
|
||||
]
|
||||
controller = ProviderConversationStateController(
|
||||
provider=provider,
|
||||
model="gpt-5.6",
|
||||
messages=messages,
|
||||
)
|
||||
state = _state("first")
|
||||
|
||||
controller.prepare_request(messages, context_window_tokens=200_000)
|
||||
response = LLMResponse(content=None, provider_state=state)
|
||||
controller.observe_response(response, messages)
|
||||
messages.extend([
|
||||
controller.project_response_message(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function"}],
|
||||
},
|
||||
response,
|
||||
),
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": "raw oversized result",
|
||||
},
|
||||
])
|
||||
governed_messages = [
|
||||
messages[0],
|
||||
messages[1],
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": "compacted result",
|
||||
},
|
||||
]
|
||||
|
||||
provider_context = controller.prepare_request(
|
||||
messages,
|
||||
context_window_tokens=200_000,
|
||||
model_messages=governed_messages,
|
||||
)
|
||||
|
||||
assert provider_context is not None
|
||||
assert provider_context.conversation_state is not None
|
||||
assert provider_context.conversation_state.pending_messages == [{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": "compacted result",
|
||||
}]
|
||||
assert controller.checkpoint(messages).pending_messages[-1]["content"] == (
|
||||
"raw oversized result"
|
||||
)
|
||||
governed_checkpoint = controller.checkpoint(
|
||||
messages,
|
||||
model_messages=governed_messages,
|
||||
)
|
||||
assert governed_checkpoint is not None
|
||||
assert governed_checkpoint.pending_messages[-1]["content"] == "compacted result"
|
||||
|
||||
|
||||
def test_transient_response_preserves_only_durable_request_messages() -> None:
|
||||
provider = _provider()
|
||||
current_message = {"role": "user", "content": "continue"}
|
||||
supplemental = {"role": "user", "content": "internal finalization retry"}
|
||||
messages = [{"role": "system", "content": "system"}, current_message]
|
||||
controller = ProviderConversationStateController(
|
||||
provider=provider,
|
||||
model="gpt-5.6",
|
||||
messages=messages,
|
||||
state=_state("saved", pending=[
|
||||
{"role": "tool", "content": "prior"},
|
||||
current_message,
|
||||
]),
|
||||
)
|
||||
|
||||
provider_context = controller.prepare_request(
|
||||
messages,
|
||||
context_window_tokens=200_000,
|
||||
supplemental_messages=[supplemental],
|
||||
)
|
||||
assert provider_context is not None
|
||||
assert provider_context.conversation_state is not None
|
||||
assert provider_context.conversation_state.pending_messages == [
|
||||
{"role": "tool", "content": "prior"},
|
||||
current_message,
|
||||
supplemental,
|
||||
]
|
||||
|
||||
controller.observe_response(
|
||||
LLMResponse(
|
||||
content="temporary failure",
|
||||
finish_reason="error",
|
||||
error_kind="timeout",
|
||||
),
|
||||
messages,
|
||||
)
|
||||
placeholder = {"role": "assistant", "content": "model error"}
|
||||
messages.append(placeholder)
|
||||
|
||||
state = controller.finish(messages)
|
||||
assert state is not None
|
||||
assert state.pending_messages == [
|
||||
{"role": "tool", "content": "prior"},
|
||||
current_message,
|
||||
placeholder,
|
||||
]
|
||||
|
||||
|
||||
def test_non_retryable_response_discards_saved_state() -> None:
|
||||
provider = _provider()
|
||||
messages = [{"role": "user", "content": "continue"}]
|
||||
controller = ProviderConversationStateController(
|
||||
provider=provider,
|
||||
model="gpt-5.6",
|
||||
messages=messages,
|
||||
state=_state("saved"),
|
||||
)
|
||||
|
||||
controller.prepare_request(messages, context_window_tokens=200_000)
|
||||
controller.observe_response(
|
||||
LLMResponse(
|
||||
content="invalid request",
|
||||
finish_reason="error",
|
||||
error_status_code=400,
|
||||
error_should_retry=False,
|
||||
),
|
||||
messages,
|
||||
)
|
||||
|
||||
assert controller.finish(messages) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("finish_reason", "exposes_tool_call"),
|
||||
[
|
||||
("length", False),
|
||||
("length", True),
|
||||
("refusal", True),
|
||||
("content_filter", True),
|
||||
],
|
||||
)
|
||||
def test_terminal_response_discards_candidate_state(
|
||||
finish_reason: str,
|
||||
exposes_tool_call: bool,
|
||||
) -> None:
|
||||
provider = _provider()
|
||||
messages = [{"role": "user", "content": "continue"}]
|
||||
controller = ProviderConversationStateController(
|
||||
provider=provider,
|
||||
model="gpt-5.6",
|
||||
messages=messages,
|
||||
state=_state("saved"),
|
||||
)
|
||||
|
||||
controller.prepare_request(messages, context_window_tokens=200_000)
|
||||
candidate = ProviderConversationState(
|
||||
kind="openai_responses",
|
||||
provider="openai:test",
|
||||
model="gpt-5.6",
|
||||
version=1,
|
||||
payload={
|
||||
"items": [{
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "exec",
|
||||
"arguments": "{}",
|
||||
}],
|
||||
},
|
||||
)
|
||||
response = LLMResponse(
|
||||
content="terminal response",
|
||||
tool_calls=(
|
||||
[ToolCallRequest(id="call_1", name="exec", arguments={})]
|
||||
if exposes_tool_call
|
||||
else []
|
||||
),
|
||||
finish_reason=finish_reason,
|
||||
provider_state=candidate,
|
||||
)
|
||||
assert response.has_tool_calls is exposes_tool_call
|
||||
assert response.should_execute_tools is False
|
||||
|
||||
controller.observe_response(response, messages)
|
||||
|
||||
assert controller.finish(messages) is None
|
||||
|
||||
|
||||
def test_independent_request_exposes_context_without_capability_check() -> None:
|
||||
provider = _provider(compact=False)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
controller = ProviderConversationStateController(
|
||||
provider=provider,
|
||||
model="gpt-5.6",
|
||||
messages=messages,
|
||||
state=_state("saved"),
|
||||
)
|
||||
|
||||
provider_context = controller.independent_request_context(
|
||||
context_window_tokens=200_000,
|
||||
)
|
||||
assert provider_context is not None
|
||||
assert provider_context.conversation_state is None
|
||||
assert provider_context.context_window_tokens == 200_000
|
||||
provider.supports_native_compaction.assert_not_called()
|
||||
@@ -10,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from nanobot.providers.base import ProviderCallContext
|
||||
from nanobot.providers.openai_compat_provider import OpenAICompatProvider
|
||||
from nanobot.providers.registry import find_by_name
|
||||
|
||||
@@ -44,8 +45,10 @@ def test_build_responses_body_strips_github_copilot_prefix():
|
||||
temperature=0.1,
|
||||
reasoning_effort=None,
|
||||
tool_choice=None,
|
||||
provider_context=ProviderCallContext(context_window_tokens=128_000),
|
||||
)
|
||||
assert body["model"] == "gpt-5.4-mini"
|
||||
assert "context_management" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from nanobot.providers.base import ProviderCallContext
|
||||
from nanobot.providers.openai_compat_provider import OpenAICompatProvider
|
||||
from nanobot.providers.registry import find_by_name
|
||||
|
||||
@@ -679,6 +680,7 @@ async def test_direct_openai_gpt5_uses_responses_api() -> None:
|
||||
assert call_kwargs["max_output_tokens"] == 4096
|
||||
assert "input" in call_kwargs
|
||||
assert "messages" not in call_kwargs
|
||||
assert call_kwargs["include"] == ["reasoning.encrypted_content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -710,6 +712,40 @@ async def test_direct_openai_reasoning_prefers_responses_api() -> None:
|
||||
assert call_kwargs["include"] == ["reasoning.encrypted_content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_openai_retries_without_unsupported_server_compaction() -> None:
|
||||
mock_chat = AsyncMock(return_value=_fake_chat_response())
|
||||
mock_responses = AsyncMock(side_effect=[
|
||||
_FakeResponsesError(400, "Unknown parameter: context_management"),
|
||||
_fake_responses_response("compaction fallback"),
|
||||
])
|
||||
spec = find_by_name("openai")
|
||||
|
||||
with patch("nanobot.providers.openai_compat_provider.AsyncOpenAI") as mock_client_class:
|
||||
client_instance = mock_client_class.return_value
|
||||
client_instance.chat.completions.create = mock_chat
|
||||
client_instance.responses.create = mock_responses
|
||||
provider = OpenAICompatProvider(
|
||||
api_key="sk-test-key",
|
||||
default_model="gpt-5.6",
|
||||
spec=spec,
|
||||
)
|
||||
|
||||
result = await provider.chat_with_context(
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
model="gpt-5.6",
|
||||
provider_context=ProviderCallContext(context_window_tokens=200_000),
|
||||
)
|
||||
|
||||
assert result.content == "compaction fallback"
|
||||
assert result.provider_state is not None
|
||||
assert mock_responses.await_count == 2
|
||||
assert "context_management" in mock_responses.call_args_list[0].kwargs
|
||||
assert "context_management" not in mock_responses.call_args_list[1].kwargs
|
||||
assert provider.supports_native_compaction("gpt-5.6") is False
|
||||
mock_chat.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_openai_gpt4o_stays_on_chat_completions() -> None:
|
||||
mock_chat = AsyncMock(return_value=_fake_chat_response())
|
||||
|
||||
@@ -20,6 +20,7 @@ from nanobot.providers.openai_codex_provider import (
|
||||
_request_codex,
|
||||
_should_retry_status,
|
||||
)
|
||||
from nanobot.providers.openai_responses import build_responses_state
|
||||
from nanobot.providers.registry import find_by_name
|
||||
|
||||
|
||||
@@ -115,6 +116,48 @@ async def test_codex_request_non_200_populates_http_metadata(monkeypatch) -> Non
|
||||
assert error.should_retry is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_request_marks_rejected_compaction_without_retaining_raw_body(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
original_client = httpx.AsyncClient
|
||||
secret = "PRIVATE PROMPT MUST NOT BE RETAINED"
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
400,
|
||||
json={
|
||||
"error": {
|
||||
"message": f"Unknown input type compaction_trigger; {secret}",
|
||||
},
|
||||
},
|
||||
request=request,
|
||||
)
|
||||
|
||||
def fake_client(
|
||||
*,
|
||||
timeout: int,
|
||||
verify: bool,
|
||||
**_kwargs: object,
|
||||
) -> httpx.AsyncClient:
|
||||
return original_client(transport=httpx.MockTransport(handler), timeout=timeout)
|
||||
|
||||
monkeypatch.setattr("nanobot.providers.openai_codex_provider.httpx.AsyncClient", fake_client)
|
||||
|
||||
with pytest.raises(_CodexHTTPError) as caught:
|
||||
await _request_codex(
|
||||
"https://codex.example/responses",
|
||||
{},
|
||||
{"input": [{"type": "compaction_trigger"}]},
|
||||
verify=True,
|
||||
)
|
||||
|
||||
error = caught.value
|
||||
assert error.compaction_unsupported is True
|
||||
assert secret not in str(error)
|
||||
assert not hasattr(error, "body")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_request_honors_stream_idle_timeout_env(monkeypatch) -> None:
|
||||
"""NANOBOT_STREAM_IDLE_TIMEOUT_S overrides the default Codex stream timeout."""
|
||||
@@ -192,7 +235,7 @@ async def test_codex_prompt_cache_key_uses_stable_conversation_prefix(monkeypatc
|
||||
):
|
||||
_ = proxy, on_thinking_delta, on_tool_call_delta
|
||||
bodies.append(body)
|
||||
return "ok", [], "stop", {}, None
|
||||
return provider_base.LLMResponse(content="ok")
|
||||
|
||||
monkeypatch.setattr("nanobot.providers.openai_codex_provider._request_codex", fake_request)
|
||||
|
||||
@@ -232,7 +275,7 @@ async def test_codex_provider_applies_extra_body_from_config(monkeypatch) -> Non
|
||||
|
||||
async def fake_request(_url, _headers, body, **_kwargs):
|
||||
bodies.append(body)
|
||||
return "ok", [], "stop", {}, None
|
||||
return provider_base.LLMResponse(content="ok")
|
||||
|
||||
monkeypatch.setattr("nanobot.providers.openai_codex_provider._request_codex", fake_request)
|
||||
config = Config.model_validate({
|
||||
@@ -297,7 +340,7 @@ async def test_codex_provider_passes_proxy_to_oauth_and_response_request(monkeyp
|
||||
):
|
||||
_ = url, headers, body, verify, on_content_delta, on_thinking_delta, on_tool_call_delta
|
||||
seen["request_proxy"] = proxy
|
||||
return "ok", [], "stop", {}, None
|
||||
return provider_base.LLMResponse(content="ok")
|
||||
|
||||
monkeypatch.setattr("nanobot.providers.openai_codex_provider.get_codex_token", fake_token)
|
||||
monkeypatch.setattr("nanobot.providers.openai_codex_provider._request_codex", fake_request)
|
||||
@@ -384,7 +427,7 @@ async def test_codex_retry_uses_structured_timeout_metadata(monkeypatch) -> None
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise httpx.ReadTimeout("")
|
||||
return "ok", [], "stop", {}, None
|
||||
return provider_base.LLMResponse(content="ok")
|
||||
|
||||
async def fake_sleep(delay: float) -> None:
|
||||
delays.append(delay)
|
||||
@@ -533,6 +576,254 @@ def test_codex_reasoning_options_request_summary_without_forcing_effort() -> Non
|
||||
assert _build_reasoning_options("none") == {"effort": "none"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_replayed_tool_turn_omits_server_item_ids(monkeypatch) -> None:
|
||||
_mock_codex_token(monkeypatch)
|
||||
provider = OpenAICodexProvider(default_model="openai-codex/gpt-5.6-sol")
|
||||
state = build_responses_state(
|
||||
provider=provider._responses_state_provider(),
|
||||
model="gpt-5.6-sol",
|
||||
input_items=[{
|
||||
"id": "msg_user",
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "Check the weather"}],
|
||||
}],
|
||||
output_items=[
|
||||
{
|
||||
"id": "rs_reasoning",
|
||||
"type": "reasoning",
|
||||
"encrypted_content": "opaque reasoning",
|
||||
"summary": [],
|
||||
},
|
||||
{
|
||||
"id": "fc_read",
|
||||
"type": "function_call",
|
||||
"call_id": "call_read",
|
||||
"name": "read_file",
|
||||
"arguments": '{"path":"weather/SKILL.md"}',
|
||||
"status": "completed",
|
||||
},
|
||||
],
|
||||
)
|
||||
bodies: list[dict[str, Any]] = []
|
||||
|
||||
async def fake_request(
|
||||
url,
|
||||
headers,
|
||||
body,
|
||||
verify,
|
||||
proxy=None,
|
||||
on_content_delta=None,
|
||||
on_thinking_delta=None,
|
||||
on_tool_call_delta=None,
|
||||
):
|
||||
bodies.append(body)
|
||||
return provider_base.LLMResponse(content="done")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"nanobot.providers.openai_codex_provider._request_codex",
|
||||
fake_request,
|
||||
)
|
||||
|
||||
response = await provider.chat(
|
||||
[{"role": "user", "content": "Check the weather"}],
|
||||
provider_context=provider_base.ProviderCallContext(
|
||||
conversation_state=state.with_pending_messages([{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_read|fc_read",
|
||||
"content": "weather skill contents",
|
||||
}]),
|
||||
),
|
||||
)
|
||||
|
||||
assert response.content == "done"
|
||||
assert len(bodies) == 1
|
||||
input_items = bodies[0]["input"]
|
||||
assert [item.get("type") for item in input_items] == [
|
||||
"message",
|
||||
"reasoning",
|
||||
"function_call",
|
||||
"function_call_output",
|
||||
]
|
||||
assert all("id" not in item for item in input_items)
|
||||
assert input_items[1]["encrypted_content"] == "opaque reasoning"
|
||||
assert input_items[2]["call_id"] == "call_read"
|
||||
assert input_items[3]["call_id"] == "call_read"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_compacts_state_at_ninety_percent_before_next_request(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
_mock_codex_token(monkeypatch)
|
||||
provider = OpenAICodexProvider(default_model="openai-codex/gpt-5.6-sol")
|
||||
state_provider = provider._responses_state_provider()
|
||||
state = build_responses_state(
|
||||
provider=state_provider,
|
||||
model="gpt-5.6-sol",
|
||||
input_items=[{"type": "message", "role": "user", "content": "old question"}],
|
||||
output_items=[
|
||||
{"type": "reasoning", "encrypted_content": "old opaque reasoning"},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "old answer"}],
|
||||
},
|
||||
],
|
||||
usage={
|
||||
"prompt_tokens": 90,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 95,
|
||||
},
|
||||
)
|
||||
bodies: list[dict[str, Any]] = []
|
||||
|
||||
async def fake_request(
|
||||
url,
|
||||
headers,
|
||||
body,
|
||||
verify,
|
||||
proxy=None,
|
||||
on_content_delta=None,
|
||||
on_thinking_delta=None,
|
||||
on_tool_call_delta=None,
|
||||
):
|
||||
_ = (
|
||||
url,
|
||||
headers,
|
||||
verify,
|
||||
proxy,
|
||||
on_content_delta,
|
||||
on_thinking_delta,
|
||||
on_tool_call_delta,
|
||||
)
|
||||
bodies.append(body)
|
||||
if body["input"][-1].get("type") == "compaction_trigger":
|
||||
compact_item = {
|
||||
"type": "compaction",
|
||||
"encrypted_content": "compacted opaque state",
|
||||
}
|
||||
return provider_base.LLMResponse(
|
||||
content=None,
|
||||
provider_state=build_responses_state(
|
||||
provider=state_provider,
|
||||
model="gpt-5.6-sol",
|
||||
input_items=body["input"],
|
||||
output_items=[compact_item],
|
||||
usage={
|
||||
"prompt_tokens": 95,
|
||||
"completion_tokens": 2,
|
||||
"total_tokens": 97,
|
||||
},
|
||||
),
|
||||
)
|
||||
return provider_base.LLMResponse(content="done")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"nanobot.providers.openai_codex_provider._request_codex",
|
||||
fake_request,
|
||||
)
|
||||
|
||||
response = await provider.chat_with_retry(
|
||||
[
|
||||
{"role": "system", "content": "system"},
|
||||
{"role": "user", "content": "new question"},
|
||||
],
|
||||
max_tokens=5,
|
||||
provider_context=provider_base.ProviderCallContext(
|
||||
conversation_state=state.with_pending_messages([
|
||||
{"role": "user", "content": "new question"},
|
||||
]),
|
||||
context_window_tokens=100,
|
||||
),
|
||||
)
|
||||
|
||||
assert response.content == "done"
|
||||
assert len(bodies) == 2
|
||||
assert bodies[0]["input"][-1] == {"type": "compaction_trigger"}
|
||||
assert bodies[1]["input"][-1] == {
|
||||
"type": "compaction",
|
||||
"encrypted_content": "compacted opaque state",
|
||||
}
|
||||
assert not any(
|
||||
item.get("type") == "reasoning"
|
||||
for item in bodies[1]["input"]
|
||||
)
|
||||
assert any(
|
||||
item.get("role") == "user"
|
||||
and "new question" in str(item.get("content"))
|
||||
for item in bodies[1]["input"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_disables_unsupported_native_compaction_and_continues(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
_mock_codex_token(monkeypatch)
|
||||
provider = OpenAICodexProvider(default_model="openai-codex/gpt-5.6-sol")
|
||||
state_provider = provider._responses_state_provider()
|
||||
state = build_responses_state(
|
||||
provider=state_provider,
|
||||
model="gpt-5.6-sol",
|
||||
input_items=[{"type": "message", "role": "user", "content": "old"}],
|
||||
output_items=[{"type": "reasoning", "encrypted_content": "opaque"}],
|
||||
usage={"prompt_tokens": 90, "completion_tokens": 5, "total_tokens": 95},
|
||||
)
|
||||
bodies: list[dict[str, Any]] = []
|
||||
|
||||
async def fake_request(
|
||||
url,
|
||||
headers,
|
||||
body,
|
||||
verify,
|
||||
proxy=None,
|
||||
on_content_delta=None,
|
||||
on_thinking_delta=None,
|
||||
on_tool_call_delta=None,
|
||||
):
|
||||
_ = (
|
||||
url,
|
||||
headers,
|
||||
verify,
|
||||
proxy,
|
||||
on_content_delta,
|
||||
on_thinking_delta,
|
||||
on_tool_call_delta,
|
||||
)
|
||||
bodies.append(body)
|
||||
if body["input"][-1].get("type") == "compaction_trigger":
|
||||
raise _CodexHTTPError(
|
||||
"HTTP 400: Codex API request failed",
|
||||
status_code=400,
|
||||
compaction_unsupported=True,
|
||||
)
|
||||
return provider_base.LLMResponse(content="done")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"nanobot.providers.openai_codex_provider._request_codex",
|
||||
fake_request,
|
||||
)
|
||||
|
||||
response = await provider.chat(
|
||||
[{"role": "user", "content": "new"}],
|
||||
max_tokens=5,
|
||||
provider_context=provider_base.ProviderCallContext(
|
||||
conversation_state=state.with_pending_messages([
|
||||
{"role": "user", "content": "new"},
|
||||
]),
|
||||
context_window_tokens=100,
|
||||
),
|
||||
)
|
||||
|
||||
assert response.content == "done"
|
||||
assert len(bodies) == 2
|
||||
assert bodies[0]["input"][-1] == {"type": "compaction_trigger"}
|
||||
assert bodies[1]["input"][-1] != {"type": "compaction_trigger"}
|
||||
assert provider.supports_native_compaction() is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_stream_surfaces_reasoning_summary(monkeypatch) -> None:
|
||||
def fake_token(**_kwargs):
|
||||
@@ -559,7 +850,12 @@ async def test_codex_stream_surfaces_reasoning_summary(monkeypatch) -> None:
|
||||
await on_content_delta("answer")
|
||||
if on_thinking_delta:
|
||||
await on_thinking_delta("summary")
|
||||
return "answer", [], "stop", {"prompt_tokens": 10, "completion_tokens": 5}, "summary"
|
||||
return provider_base.LLMResponse(
|
||||
content="answer",
|
||||
finish_reason="stop",
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 5},
|
||||
reasoning_content="summary",
|
||||
)
|
||||
|
||||
monkeypatch.setattr("nanobot.providers.openai_codex_provider._request_codex", fake_request)
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Tests for the shared openai_responses converters and parsers."""
|
||||
|
||||
import json
|
||||
from io import StringIO
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from loguru import logger
|
||||
|
||||
from nanobot.providers.openai_responses.converters import (
|
||||
convert_messages,
|
||||
@@ -12,12 +14,22 @@ from nanobot.providers.openai_responses.converters import (
|
||||
split_tool_call_id,
|
||||
)
|
||||
from nanobot.providers.openai_responses.parsing import (
|
||||
ResponsesStreamCapture,
|
||||
consume_sdk_stream,
|
||||
consume_sse,
|
||||
consume_sse_with_reasoning,
|
||||
is_replayable_finish_reason,
|
||||
map_finish_reason,
|
||||
parse_response_output,
|
||||
)
|
||||
from nanobot.providers.openai_responses.state import (
|
||||
build_responses_state,
|
||||
is_compaction_compatibility_error,
|
||||
prepare_responses_input,
|
||||
resolve_compact_threshold,
|
||||
responses_state_context_tokens,
|
||||
responses_state_items,
|
||||
)
|
||||
|
||||
# ======================================================================
|
||||
# converters - split_tool_call_id
|
||||
@@ -398,6 +410,17 @@ class TestMapFinishReason:
|
||||
def test_unknown_defaults_to_stop(self):
|
||||
assert map_finish_reason("some_new_status") == "stop"
|
||||
|
||||
@pytest.mark.parametrize("finish_reason", ["stop", "tool_calls", "function_call"])
|
||||
def test_replayable_finish_reasons(self, finish_reason):
|
||||
assert is_replayable_finish_reason(finish_reason) is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"finish_reason",
|
||||
["length", "refusal", "content_filter", "error"],
|
||||
)
|
||||
def test_non_replayable_finish_reasons(self, finish_reason):
|
||||
assert is_replayable_finish_reason(finish_reason) is False
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# parsing - parse_response_output
|
||||
@@ -418,6 +441,29 @@ class TestParseResponseOutput:
|
||||
assert result.usage == {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
assert result.tool_calls == []
|
||||
|
||||
def test_refusal_response_surfaces_text_without_advancing_state(self):
|
||||
refusal = "I can’t help with that request."
|
||||
resp = {
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "refusal", "refusal": refusal}],
|
||||
}],
|
||||
"status": "completed",
|
||||
"usage": {},
|
||||
}
|
||||
|
||||
result = parse_response_output(
|
||||
resp,
|
||||
state_provider="openai:test",
|
||||
state_model="gpt-5.6",
|
||||
state_input_items=[{"role": "user", "content": "request"}],
|
||||
)
|
||||
|
||||
assert result.content == refusal
|
||||
assert result.finish_reason == "refusal"
|
||||
assert result.provider_state is None
|
||||
|
||||
def test_tool_call_response(self):
|
||||
resp = {
|
||||
"output": [{
|
||||
@@ -429,12 +475,18 @@ class TestParseResponseOutput:
|
||||
"status": "completed",
|
||||
"usage": {},
|
||||
}
|
||||
result = parse_response_output(resp)
|
||||
result = parse_response_output(
|
||||
resp,
|
||||
state_provider="openai:test",
|
||||
state_model="gpt-5.6",
|
||||
state_input_items=[{"role": "user", "content": "weather?"}],
|
||||
)
|
||||
assert result.content is None
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].name == "get_weather"
|
||||
assert result.tool_calls[0].arguments == {"city": "SF"}
|
||||
assert result.tool_calls[0].id == "call_1|fc_1"
|
||||
assert result.provider_state is not None
|
||||
|
||||
def test_malformed_tool_arguments_logged(self):
|
||||
"""Malformed JSON arguments should log a warning and remain non-object."""
|
||||
@@ -493,10 +545,39 @@ class TestParseResponseOutput:
|
||||
assert result.content is None
|
||||
assert result.tool_calls == []
|
||||
|
||||
def test_incomplete_status(self):
|
||||
resp = {"output": [], "status": "incomplete", "usage": {}}
|
||||
result = parse_response_output(resp)
|
||||
assert result.finish_reason == "length"
|
||||
@pytest.mark.parametrize(
|
||||
("reason", "expected_finish_reason"),
|
||||
[
|
||||
("max_output_tokens", "length"),
|
||||
("content_filter", "content_filter"),
|
||||
],
|
||||
)
|
||||
def test_incomplete_status(self, reason, expected_finish_reason):
|
||||
resp = {
|
||||
"output": [],
|
||||
"status": "incomplete",
|
||||
"incomplete_details": {"reason": reason},
|
||||
"usage": {},
|
||||
}
|
||||
result = parse_response_output(
|
||||
resp,
|
||||
state_provider="openai:test",
|
||||
state_model="gpt-5.6",
|
||||
state_input_items=[{"role": "user", "content": "prompt"}],
|
||||
)
|
||||
assert result.finish_reason == expected_finish_reason
|
||||
assert result.provider_state is None
|
||||
|
||||
def test_unknown_status_does_not_advance_provider_state(self):
|
||||
result = parse_response_output(
|
||||
{"output": [], "status": "future_terminal_status", "usage": {}},
|
||||
state_provider="openai:test",
|
||||
state_model="gpt-5.6",
|
||||
state_input_items=[{"role": "user", "content": "prompt"}],
|
||||
)
|
||||
|
||||
assert result.finish_reason == "stop"
|
||||
assert result.provider_state is None
|
||||
|
||||
def test_sdk_model_object(self):
|
||||
"""parse_response_output should handle SDK objects with model_dump()."""
|
||||
@@ -523,6 +604,194 @@ class TestParseResponseOutput:
|
||||
assert result.usage["completion_tokens"] == 50
|
||||
assert result.usage["total_tokens"] == 150
|
||||
|
||||
def test_preserves_every_output_item_as_opaque_state(self):
|
||||
input_items = [{"role": "user", "content": "inspect the repo"}]
|
||||
output = [
|
||||
{
|
||||
"id": "rs_1",
|
||||
"type": "reasoning",
|
||||
"encrypted_content": "opaque-secret",
|
||||
"summary": [],
|
||||
},
|
||||
{
|
||||
"id": "future_1",
|
||||
"type": "future_item_type",
|
||||
"provider_field": {"nested": True},
|
||||
},
|
||||
{
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "done"}],
|
||||
},
|
||||
]
|
||||
|
||||
result = parse_response_output(
|
||||
{"output": output, "status": "completed", "usage": {}},
|
||||
state_provider="openai:test",
|
||||
state_model="gpt-5.6",
|
||||
state_input_items=input_items,
|
||||
)
|
||||
|
||||
assert result.provider_state is not None
|
||||
assert responses_state_items(result.provider_state) == [*input_items, *output]
|
||||
|
||||
|
||||
class TestResponsesConversationState:
|
||||
def test_server_compaction_prunes_superseded_prefix(self):
|
||||
state = build_responses_state(
|
||||
provider="openai:test",
|
||||
model="gpt-5.6",
|
||||
input_items=[
|
||||
{"type": "message", "role": "user", "content": "old"},
|
||||
{"type": "reasoning", "encrypted_content": "old-reasoning"},
|
||||
],
|
||||
output_items=[
|
||||
{"type": "compaction", "encrypted_content": "compact"},
|
||||
{"type": "message", "role": "assistant", "content": "new"},
|
||||
],
|
||||
usage={
|
||||
"prompt_tokens": 90,
|
||||
"completion_tokens": 10,
|
||||
"total_tokens": 100,
|
||||
},
|
||||
)
|
||||
|
||||
assert responses_state_items(state) == [
|
||||
{"type": "compaction", "encrypted_content": "compact"},
|
||||
{"type": "message", "role": "assistant", "content": "new"},
|
||||
]
|
||||
assert responses_state_context_tokens(state) == 100
|
||||
|
||||
def test_existing_compaction_keeps_canonical_retained_prefix(self):
|
||||
canonical_input = [
|
||||
{"type": "message", "role": "user", "content": "retained"},
|
||||
{"type": "compaction", "encrypted_content": "compact"},
|
||||
]
|
||||
output = [{"type": "message", "role": "assistant", "content": "new"}]
|
||||
|
||||
state = build_responses_state(
|
||||
provider="openai:test",
|
||||
model="gpt-5.6",
|
||||
input_items=canonical_input,
|
||||
output_items=output,
|
||||
)
|
||||
|
||||
assert responses_state_items(state) == [*canonical_input, *output]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("context_window", "max_output", "expected"),
|
||||
[
|
||||
(200_000, 20_000, 180_000),
|
||||
(100_000, 30_000, 70_000),
|
||||
(0, 4_096, None),
|
||||
],
|
||||
)
|
||||
def test_compact_threshold_reserves_codex_style_headroom(
|
||||
self,
|
||||
context_window,
|
||||
max_output,
|
||||
expected,
|
||||
):
|
||||
assert resolve_compact_threshold(context_window, max_output) == expected
|
||||
|
||||
def test_compaction_compatibility_recognizes_old_sdk_signature_error(self):
|
||||
error = TypeError("create() got an unexpected keyword argument 'context_management'")
|
||||
assert is_compaction_compatibility_error(error) is True
|
||||
assert is_compaction_compatibility_error(TypeError("unrelated argument")) is False
|
||||
|
||||
def test_state_observability_logs_counts_without_opaque_content(self):
|
||||
secret = "opaque-secret-that-must-not-be-logged"
|
||||
state = build_responses_state(
|
||||
provider=f"openai:https://example.test/?key={secret}",
|
||||
model=f"secret-model-{secret}",
|
||||
input_items=[{"role": "user", "content": secret}],
|
||||
output_items=[{"type": "reasoning", "encrypted_content": secret}],
|
||||
).with_pending_messages([{"role": "user", "content": secret}])
|
||||
sink = StringIO()
|
||||
sink_id = logger.add(sink, level="DEBUG", format="{message}")
|
||||
try:
|
||||
prepare_responses_input(
|
||||
[{"role": "user", "content": secret}],
|
||||
state=state,
|
||||
provider=state.provider,
|
||||
model=state.model,
|
||||
)
|
||||
build_responses_state(
|
||||
provider=state.provider,
|
||||
model=state.model,
|
||||
input_items=[
|
||||
{"role": "user", "content": secret},
|
||||
{"type": "reasoning", "encrypted_content": secret},
|
||||
],
|
||||
output_items=[
|
||||
{"type": "compaction", "encrypted_content": secret},
|
||||
],
|
||||
)
|
||||
finally:
|
||||
logger.remove(sink_id)
|
||||
|
||||
log_text = sink.getvalue()
|
||||
assert "prior_items=2" in log_text
|
||||
assert "pending_messages=1" in log_text
|
||||
assert "dropped_items=2" in log_text
|
||||
assert secret not in log_text
|
||||
|
||||
def test_replays_exact_items_then_only_pending_and_new_messages(self):
|
||||
prior_items = [
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_1",
|
||||
"encrypted_content": "opaque-secret",
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "fc_1",
|
||||
"call_id": "call_1",
|
||||
"name": "read_file",
|
||||
"arguments": '{"path":"a.py"}',
|
||||
},
|
||||
]
|
||||
state = build_responses_state(
|
||||
provider="openai:test",
|
||||
model="gpt-5.6",
|
||||
input_items=prior_items[:1],
|
||||
output_items=prior_items[1:],
|
||||
).with_pending_messages([
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1|fc_1",
|
||||
"content": "file contents",
|
||||
},
|
||||
{"role": "user", "content": "continue"},
|
||||
])
|
||||
|
||||
instructions, items, replayed = prepare_responses_input(
|
||||
[
|
||||
{"role": "system", "content": "current instructions"},
|
||||
{"role": "user", "content": "a lossy public transcript"},
|
||||
],
|
||||
state=state,
|
||||
provider="openai:test",
|
||||
model="gpt-5.6",
|
||||
)
|
||||
|
||||
assert instructions == "current instructions"
|
||||
assert replayed is True
|
||||
assert items[:3] == prior_items
|
||||
assert items[3] == {
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": "file contents",
|
||||
}
|
||||
assert items[4] == {
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "continue"}],
|
||||
}
|
||||
assert "lossy public transcript" not in str(items)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# parsing - consume_sse
|
||||
@@ -553,6 +822,122 @@ class TestConsumeSse:
|
||||
assert tool_calls == []
|
||||
assert finish_reason == "stop"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refusal_events_reconcile_parts_and_terminal_output(self):
|
||||
refusal = "First and second sentence. Done-only. Terminal suffix."
|
||||
terminal_response = {
|
||||
"status": "completed",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "msg_2",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "refusal", "refusal": refusal}],
|
||||
}],
|
||||
}
|
||||
response = _SseResponse([
|
||||
{
|
||||
"type": "response.refusal.delta",
|
||||
"item_id": "msg_1",
|
||||
"content_index": 0,
|
||||
"delta": "First",
|
||||
},
|
||||
{
|
||||
"type": "response.refusal.delta",
|
||||
"item_id": "msg_1",
|
||||
"content_index": 1,
|
||||
"delta": " and second",
|
||||
},
|
||||
{
|
||||
"type": "response.refusal.done",
|
||||
"item_id": "msg_1",
|
||||
"content_index": 0,
|
||||
"refusal": "First",
|
||||
},
|
||||
{
|
||||
"type": "response.refusal.done",
|
||||
"item_id": "msg_1",
|
||||
"content_index": 1,
|
||||
"refusal": " and second sentence.",
|
||||
},
|
||||
{
|
||||
"type": "response.refusal.done",
|
||||
"item_id": "msg_2",
|
||||
"content_index": 0,
|
||||
"refusal": " Done-only.",
|
||||
},
|
||||
{
|
||||
"type": "response.refusal.delta",
|
||||
"item_id": "msg_2",
|
||||
"content_index": 1,
|
||||
"delta": " Terminal",
|
||||
},
|
||||
{"type": "response.completed", "response": terminal_response},
|
||||
])
|
||||
capture = ResponsesStreamCapture()
|
||||
deltas: list[str] = []
|
||||
|
||||
async def on_content(delta: str) -> None:
|
||||
deltas.append(delta)
|
||||
|
||||
content, _, finish_reason, _, _ = await consume_sse_with_reasoning(
|
||||
response,
|
||||
on_content_delta=on_content,
|
||||
capture=capture,
|
||||
)
|
||||
|
||||
assert content == refusal
|
||||
assert deltas == [
|
||||
"First",
|
||||
" and second",
|
||||
" sentence.",
|
||||
" Done-only.",
|
||||
" Terminal",
|
||||
" suffix.",
|
||||
]
|
||||
assert finish_reason == "refusal"
|
||||
assert capture.completed is True
|
||||
assert is_replayable_finish_reason(finish_reason) is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("source", ["events", "terminal"])
|
||||
async def test_refusal_without_deltas_has_non_replayable_finish(self, source: str):
|
||||
refusal = "I can’t help with that request."
|
||||
terminal_response = {
|
||||
"status": "completed",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "refusal", "refusal": refusal}],
|
||||
}],
|
||||
}
|
||||
events = (
|
||||
[
|
||||
{"type": "response.refusal.done", "refusal": refusal},
|
||||
{"type": "response.completed", "response": {"status": "completed"}},
|
||||
]
|
||||
if source == "events"
|
||||
else [{"type": "response.completed", "response": terminal_response}]
|
||||
)
|
||||
response = _SseResponse(events)
|
||||
capture = ResponsesStreamCapture()
|
||||
deltas: list[str] = []
|
||||
|
||||
async def on_content(delta: str) -> None:
|
||||
deltas.append(delta)
|
||||
|
||||
content, _, finish_reason, _, _ = await consume_sse_with_reasoning(
|
||||
response,
|
||||
on_content_delta=on_content,
|
||||
capture=capture,
|
||||
)
|
||||
|
||||
assert content == refusal
|
||||
assert deltas == [refusal]
|
||||
assert finish_reason == "refusal"
|
||||
assert capture.completed is True
|
||||
assert is_replayable_finish_reason(finish_reason) is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_summary_delta_extracted(self):
|
||||
response = _SseResponse([
|
||||
@@ -599,6 +984,139 @@ class TestConsumeSse:
|
||||
|
||||
assert reasoning == "cached summary"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_commits_exact_items_only_after_completed_event(self):
|
||||
output = [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_1",
|
||||
"encrypted_content": "opaque-secret",
|
||||
},
|
||||
{"type": "future_item_type", "id": "future_1", "value": 7},
|
||||
]
|
||||
capture = ResponsesStreamCapture()
|
||||
response = _SseResponse([
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": output[0],
|
||||
},
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 1,
|
||||
"item": output[1],
|
||||
},
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {"status": "completed", "output": output},
|
||||
},
|
||||
])
|
||||
|
||||
await consume_sse_with_reasoning(response, capture=capture)
|
||||
|
||||
assert capture.completed is True
|
||||
assert capture.output_items == output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_keeps_done_items_when_completed_output_is_empty(self):
|
||||
output = [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_1",
|
||||
"encrypted_content": "opaque-secret",
|
||||
"summary": [],
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "fc_1",
|
||||
"call_id": "call_1",
|
||||
"name": "read_file",
|
||||
"arguments": '{"path":"weather/SKILL.md"}',
|
||||
},
|
||||
]
|
||||
capture = ResponsesStreamCapture()
|
||||
response = _SseResponse([
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": index,
|
||||
"item": item,
|
||||
}
|
||||
for index, item in enumerate(output)
|
||||
] + [{
|
||||
"type": "response.completed",
|
||||
"response": {"status": "completed", "output": []},
|
||||
}])
|
||||
|
||||
await consume_sse_with_reasoning(response, capture=capture)
|
||||
|
||||
assert capture.completed is True
|
||||
assert capture.output_items == output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("reason", "expected_finish_reason"),
|
||||
[
|
||||
("max_output_tokens", "length"),
|
||||
("content_filter", "content_filter"),
|
||||
],
|
||||
)
|
||||
async def test_incomplete_event_commits_capture_usage(
|
||||
self,
|
||||
reason,
|
||||
expected_finish_reason,
|
||||
):
|
||||
output = [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"status": "incomplete",
|
||||
"content": [{"type": "output_text", "text": "partial"}],
|
||||
},
|
||||
]
|
||||
terminal_response = {
|
||||
"id": "resp_1",
|
||||
"status": "incomplete",
|
||||
"incomplete_details": {"reason": reason},
|
||||
"output": output,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
capture = ResponsesStreamCapture()
|
||||
response = _SseResponse([
|
||||
{"type": "response.output_text.delta", "delta": "partial"},
|
||||
{"type": "response.incomplete", "response": terminal_response},
|
||||
])
|
||||
|
||||
content, _, finish_reason, usage, _ = await consume_sse_with_reasoning(
|
||||
response,
|
||||
capture=capture,
|
||||
)
|
||||
|
||||
assert content == "partial"
|
||||
assert finish_reason == expected_finish_reason
|
||||
assert usage == {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
assert capture.completed is True
|
||||
assert capture.response == terminal_response
|
||||
assert capture.output_items == output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_capture_does_not_commit_interrupted_stream(self):
|
||||
capture = ResponsesStreamCapture()
|
||||
response = _SseResponse([
|
||||
{
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": {
|
||||
"type": "reasoning",
|
||||
"id": "rs_1",
|
||||
"encrypted_content": "opaque-secret",
|
||||
},
|
||||
},
|
||||
])
|
||||
|
||||
await consume_sse_with_reasoning(response, capture=capture)
|
||||
|
||||
assert capture.completed is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_summary_from_done_item(self):
|
||||
response = _SseResponse([
|
||||
@@ -755,6 +1273,131 @@ class TestConsumeSdkStream:
|
||||
assert tool_calls == []
|
||||
assert finish_reason == "stop"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refusal_events_reconcile_parts_and_terminal_output(self):
|
||||
refusal = "First and second sentence. Done-only. Terminal suffix."
|
||||
terminal_response = {
|
||||
"status": "completed",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "msg_2",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "refusal", "refusal": refusal}],
|
||||
}],
|
||||
}
|
||||
resp_obj = MagicMock(status="completed", usage=None, output=[])
|
||||
resp_obj.model_dump.return_value = terminal_response
|
||||
events = [
|
||||
MagicMock(
|
||||
type="response.refusal.delta",
|
||||
item_id="msg_1",
|
||||
content_index=0,
|
||||
delta="First",
|
||||
),
|
||||
MagicMock(
|
||||
type="response.refusal.delta",
|
||||
item_id="msg_1",
|
||||
content_index=1,
|
||||
delta=" and second",
|
||||
),
|
||||
MagicMock(
|
||||
type="response.refusal.done",
|
||||
item_id="msg_1",
|
||||
content_index=0,
|
||||
refusal="First",
|
||||
),
|
||||
MagicMock(
|
||||
type="response.refusal.done",
|
||||
item_id="msg_1",
|
||||
content_index=1,
|
||||
refusal=" and second sentence.",
|
||||
),
|
||||
MagicMock(
|
||||
type="response.refusal.done",
|
||||
item_id="msg_2",
|
||||
content_index=0,
|
||||
refusal=" Done-only.",
|
||||
),
|
||||
MagicMock(
|
||||
type="response.refusal.delta",
|
||||
item_id="msg_2",
|
||||
content_index=1,
|
||||
delta=" Terminal",
|
||||
),
|
||||
MagicMock(type="response.completed", response=resp_obj),
|
||||
]
|
||||
capture = ResponsesStreamCapture()
|
||||
deltas: list[str] = []
|
||||
|
||||
async def on_content(delta: str) -> None:
|
||||
deltas.append(delta)
|
||||
|
||||
async def stream():
|
||||
for event in events:
|
||||
yield event
|
||||
|
||||
content, _, finish_reason, _, _ = await consume_sdk_stream(
|
||||
stream(),
|
||||
on_content_delta=on_content,
|
||||
capture=capture,
|
||||
)
|
||||
|
||||
assert content == refusal
|
||||
assert deltas == [
|
||||
"First",
|
||||
" and second",
|
||||
" sentence.",
|
||||
" Done-only.",
|
||||
" Terminal",
|
||||
" suffix.",
|
||||
]
|
||||
assert finish_reason == "refusal"
|
||||
assert capture.completed is True
|
||||
assert is_replayable_finish_reason(finish_reason) is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("source", ["events", "terminal"])
|
||||
async def test_refusal_without_deltas_has_non_replayable_finish(self, source: str):
|
||||
refusal = "I can’t help with that request."
|
||||
terminal_response = {
|
||||
"status": "completed",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "refusal", "refusal": refusal}],
|
||||
}],
|
||||
}
|
||||
resp_obj = MagicMock(status="completed", usage=None, output=[])
|
||||
resp_obj.model_dump.return_value = terminal_response
|
||||
capture = ResponsesStreamCapture()
|
||||
deltas: list[str] = []
|
||||
|
||||
async def on_content(delta: str) -> None:
|
||||
deltas.append(delta)
|
||||
|
||||
async def stream():
|
||||
if source == "events":
|
||||
yield MagicMock(type="response.refusal.done", refusal=refusal)
|
||||
yield MagicMock(
|
||||
type="response.completed",
|
||||
response={"status": "completed"},
|
||||
)
|
||||
else:
|
||||
yield MagicMock(type="response.completed", response=resp_obj)
|
||||
|
||||
content, _, finish_reason, _, _ = await consume_sdk_stream(
|
||||
stream(),
|
||||
on_content_delta=on_content,
|
||||
capture=capture,
|
||||
)
|
||||
|
||||
assert content == refusal
|
||||
assert deltas == [refusal]
|
||||
assert finish_reason == "refusal"
|
||||
assert capture.completed is True
|
||||
assert is_replayable_finish_reason(finish_reason) is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_on_content_delta_called(self):
|
||||
ev1 = MagicMock(type="response.output_text.delta", delta="hi")
|
||||
@@ -919,6 +1562,64 @@ class TestConsumeSdkStream:
|
||||
_, _, _, usage, _ = await consume_sdk_stream(stream())
|
||||
assert usage == {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("reason", "expected_finish_reason"),
|
||||
[
|
||||
("max_output_tokens", "length"),
|
||||
("content_filter", "content_filter"),
|
||||
],
|
||||
)
|
||||
async def test_incomplete_event_commits_capture_usage(
|
||||
self,
|
||||
reason,
|
||||
expected_finish_reason,
|
||||
):
|
||||
output = [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"status": "incomplete",
|
||||
"content": [{"type": "output_text", "text": "partial"}],
|
||||
},
|
||||
]
|
||||
usage_obj = MagicMock(input_tokens=10, output_tokens=5, total_tokens=15)
|
||||
output_item = MagicMock(type="message")
|
||||
terminal_response = {
|
||||
"id": "resp_1",
|
||||
"status": "incomplete",
|
||||
"incomplete_details": {"reason": reason},
|
||||
"output": output,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
resp_obj = MagicMock(
|
||||
status="incomplete",
|
||||
usage=usage_obj,
|
||||
output=[output_item],
|
||||
)
|
||||
resp_obj.model_dump.return_value = terminal_response
|
||||
events = [
|
||||
MagicMock(type="response.output_text.delta", delta="partial"),
|
||||
MagicMock(type="response.incomplete", response=resp_obj),
|
||||
]
|
||||
capture = ResponsesStreamCapture()
|
||||
|
||||
async def stream():
|
||||
for event in events:
|
||||
yield event
|
||||
|
||||
content, _, finish_reason, usage, _ = await consume_sdk_stream(
|
||||
stream(),
|
||||
capture=capture,
|
||||
)
|
||||
|
||||
assert content == "partial"
|
||||
assert finish_reason == expected_finish_reason
|
||||
assert usage == {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||||
assert capture.completed is True
|
||||
assert capture.response == terminal_response
|
||||
assert capture.output_items == output
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_extracted(self):
|
||||
summary_item = MagicMock(type="summary_text", text="thinking...")
|
||||
|
||||
@@ -3,7 +3,14 @@ import copy
|
||||
|
||||
import pytest
|
||||
|
||||
from nanobot.providers.base import RETRY_AFTER_BUFFER, GenerationSettings, LLMProvider, LLMResponse
|
||||
from nanobot.providers.base import (
|
||||
RETRY_AFTER_BUFFER,
|
||||
GenerationSettings,
|
||||
LLMProvider,
|
||||
LLMResponse,
|
||||
ProviderCallContext,
|
||||
ProviderConversationState,
|
||||
)
|
||||
|
||||
|
||||
class ScriptedProvider(LLMProvider):
|
||||
@@ -330,6 +337,79 @@ async def test_successful_image_retry_mutates_original_messages_in_place() -> No
|
||||
assert any("not delivered" in (block.get("text") or "").lower() for block in content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("messages", "payload", "pending_messages"),
|
||||
[
|
||||
(_IMAGE_MSG, {}, _IMAGE_MSG),
|
||||
(
|
||||
[{"role": "user", "content": "continue"}],
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,abc",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
[],
|
||||
),
|
||||
],
|
||||
ids=["pending-image", "opaque-payload-image"],
|
||||
)
|
||||
async def test_image_retry_discards_provider_state_with_images(
|
||||
messages,
|
||||
payload,
|
||||
pending_messages,
|
||||
) -> None:
|
||||
class ContextScriptedProvider(ScriptedProvider):
|
||||
def __init__(self, responses):
|
||||
super().__init__(responses)
|
||||
self.contexts: list[ProviderCallContext] = []
|
||||
|
||||
async def chat_with_context(
|
||||
self,
|
||||
*,
|
||||
provider_context: ProviderCallContext,
|
||||
**kwargs,
|
||||
) -> LLMResponse:
|
||||
self.contexts.append(provider_context)
|
||||
return await self.chat(**kwargs)
|
||||
|
||||
provider = ContextScriptedProvider([
|
||||
LLMResponse(content="model does not support images", finish_reason="error"),
|
||||
LLMResponse(content="ok, no image"),
|
||||
])
|
||||
messages = copy.deepcopy(messages)
|
||||
state = ProviderConversationState(
|
||||
kind="openai_responses",
|
||||
provider="openai:test",
|
||||
model="gpt-5.6",
|
||||
version=1,
|
||||
payload=copy.deepcopy(payload),
|
||||
pending_messages=copy.deepcopy(pending_messages),
|
||||
)
|
||||
|
||||
response = await provider.chat_with_retry(
|
||||
messages=messages,
|
||||
provider_context=ProviderCallContext(conversation_state=state),
|
||||
)
|
||||
|
||||
assert response.content == "ok, no image"
|
||||
retry_context = provider.contexts[-1]
|
||||
assert isinstance(retry_context, ProviderCallContext)
|
||||
assert retry_context.conversation_state is None
|
||||
public_content = messages[0]["content"]
|
||||
if isinstance(public_content, list):
|
||||
assert all(block.get("type") != "image_url" for block in public_content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_transient_error_without_images_no_retry() -> None:
|
||||
"""Non-transient errors without image content are returned immediately."""
|
||||
|
||||
@@ -4,6 +4,7 @@ import time
|
||||
|
||||
import pytest
|
||||
|
||||
from nanobot.providers.base import ProviderCallContext
|
||||
from nanobot.providers.openai_compat_provider import (
|
||||
_RESPONSES_FAILURE_THRESHOLD,
|
||||
_RESPONSES_PROBE_INTERVAL_S,
|
||||
@@ -28,6 +29,26 @@ def test_responses_api_available_by_default(provider):
|
||||
assert provider._should_use_responses_api("gpt-5", None) is True
|
||||
|
||||
|
||||
def test_direct_openai_enables_server_compaction(provider):
|
||||
provider._extra_body = {}
|
||||
|
||||
body = provider._build_responses_body(
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
model="gpt-5.6",
|
||||
max_tokens=30_000,
|
||||
temperature=0.1,
|
||||
reasoning_effort="high",
|
||||
tool_choice=None,
|
||||
provider_context=ProviderCallContext(context_window_tokens=100_000),
|
||||
)
|
||||
|
||||
assert body["context_management"] == [{
|
||||
"type": "compaction",
|
||||
"compact_threshold": 70_000,
|
||||
}]
|
||||
|
||||
|
||||
def test_api_type_chat_completions_disables_responses(provider):
|
||||
provider._api_type = "chat_completions"
|
||||
assert provider._should_use_responses_api("gpt-5", None) is False
|
||||
|
||||
Reference in New Issue
Block a user