mirror/nanobot
1
0
Fork 0
mirror of https://github.com/HKUDS/nanobot.git synced 2026-09-04 10:11:46 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(agent): let subagents recover from tool errors

Browse Source
This commit is contained in:
chengyongru
2026-08-25 11:54:40 +08:00
committed by chengyongru
parent 2e7ebeb1ca
commit 8332c604da
12 changed files with 140 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/agent/test_runner_errors.py
+33 -26
View File
@@ -9,6 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from agent.runner_helpers import make_run_spec
from nanobot.agent.tools import ToolResult
from nanobot.config.schema import AgentDefaults
from nanobot.providers.base import LLMProvider, LLMResponse, ToolCallRequest
@@ -16,14 +17,17 @@ _MAX_TOOL_RESULT_CHARS = AgentDefaults().max_tool_result_chars
@pytest.mark.asyncio
async def test_runner_returns_structured_tool_error():
async def test_runner_returns_tool_exception_to_model_for_recovery():
from nanobot.agent.runner import AgentRunner
provider = MagicMock(spec=LLMProvider)
provider.chat_with_retry = AsyncMock(return_value=LLMResponse(
content="working",
tool_calls=[ToolCallRequest(id="call_1", name="list_dir", arguments={})],
))
provider.chat_with_retry = AsyncMock(side_effect=[
LLMResponse(
content="working",
tool_calls=[ToolCallRequest(id="call_1", name="list_dir", arguments={})],
),
LLMResponse(content="recovered", tool_calls=[]),
])
tools = MagicMock()
tools.get_definitions.return_value = []
tools.execute = AsyncMock(side_effect=RuntimeError("boom"))
@@ -36,14 +40,17 @@ async def test_runner_returns_structured_tool_error():
model="test-model",
max_iterations=2,
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
fail_on_tool_error=True,
))
assert result.stop_reason == "tool_error"
assert result.error == "Error: RuntimeError: boom"
assert provider.chat_with_retry.await_count == 2
assert result.stop_reason == "completed"
assert result.error is None
assert result.final_content == "recovered"
assert result.tool_events == [
{"name": "list_dir", "status": "error", "detail": "boom"}
]
tool_message = next(message for message in result.messages if message.get("role") == "tool")
assert "Error: RuntimeError: boom" in tool_message["content"]
@pytest.mark.asyncio
@@ -180,35 +187,38 @@ async def test_runner_ignores_tool_calls_when_finish_reason_blocks_execution(
@pytest.mark.asyncio
async def test_runner_tool_error_sets_final_content():
async def test_runner_returns_structured_tool_error_to_model_for_recovery():
from nanobot.agent.runner import AgentRunner
provider = MagicMock(spec=LLMProvider)
async def chat_with_retry(*, messages, **kwargs):
return LLMResponse(
provider.chat_with_retry = AsyncMock(side_effect=[
LLMResponse(
content="working",
tool_calls=[ToolCallRequest(id="call_1", name="read_file", arguments={"path": "x"})],
usage=None,
)
provider.chat_with_retry = chat_with_retry
),
LLMResponse(content="used another path", tool_calls=[], usage=None),
])
tools = MagicMock()
tools.get_definitions.return_value = []
tools.execute = AsyncMock(side_effect=RuntimeError("boom"))
tools.execute = AsyncMock(return_value=ToolResult.error("Error: File not found: x"))
runner = AgentRunner()
result = await runner.run(make_run_spec(provider,
initial_messages=[{"role": "user", "content": "do task"}],
tools=tools,
model="test-model",
max_iterations=1,
max_iterations=2,
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
fail_on_tool_error=True,
))
assert result.final_content == "Error: RuntimeError: boom"
assert result.stop_reason == "tool_error"
assert provider.chat_with_retry.await_count == 2
assert result.final_content == "used another path"
assert result.stop_reason == "completed"
assert result.tool_events == [
{"name": "read_file", "status": "error", "detail": "Error: File not found: x"}
]
@pytest.mark.asyncio
@@ -241,7 +251,6 @@ async def test_runner_preserves_successful_exec_output_that_starts_with_error():
model="test-model",
max_iterations=2,
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
fail_on_tool_error=True,
))
assert result.final_content == "done"
@@ -252,9 +261,8 @@ async def test_runner_preserves_successful_exec_output_that_starts_with_error():
@pytest.mark.asyncio
async def test_runner_tool_error_preserves_tool_results_in_messages():
"""When a tool raises a fatal error, its results must still be appended
to messages so the session never contains orphan tool_calls (#2943)."""
async def test_runner_preserves_tool_error_results_in_messages():
"""Tool errors stay paired with their calls so the model can recover (#2943)."""
from nanobot.agent.runner import AgentRunner
provider = MagicMock(spec=LLMProvider)
@@ -292,11 +300,10 @@ async def test_runner_tool_error_preserves_tool_results_in_messages():
model="test-model",
max_iterations=1,
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
fail_on_tool_error=True,
))
assert result.stop_reason == "tool_error"
# Both tool results must be in messages even though tc2 had a fatal error.
assert result.stop_reason == "max_iterations"
# Both tool results must be in messages even though tc2 returned an error.
tool_msgs = [m for m in result.messages if m.get("role") == "tool"]
assert len(tool_msgs) == 2
assert tool_msgs[0]["tool_call_id"] == "tc1"
tests/agent/test_runner_injections.py
+3 -4
View File
@@ -1448,7 +1448,7 @@ async def test_dispatch_republishes_leftover_queue_messages(tmp_path):
"""Messages left in the pending queue after _dispatch are re-published to the bus.
This tests the finally-block cleanup that prevents message loss when
the runner exits early (e.g., max_iterations, tool_error) with messages
the runner exits early (e.g., max_iterations) with messages
still in the queue.
"""
from nanobot.bus.events import InboundMessage
@@ -1488,8 +1488,8 @@ async def test_dispatch_republishes_leftover_queue_messages(tmp_path):
@pytest.mark.asyncio
async def test_drain_injections_on_fatal_tool_error():
"""A fatal tool error must not leak recovered content into an injected follow-up."""
async def test_drain_injections_after_recoverable_tool_error():
"""A tool error and injected follow-up continue in the same runner conversation."""
from nanobot.agent.runner import AgentRunner
from nanobot.bus.events import InboundMessage
@@ -1532,7 +1532,6 @@ async def test_drain_injections_on_fatal_tool_error():
model="test-model",
max_iterations=5,
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
fail_on_tool_error=True,
injection_callback=inject_cb,
))
tests/agent/test_runner_tool_execution.py
+12 -10
View File
@@ -398,24 +398,27 @@ async def test_runner_rejects_openai_responses_array_arguments_without_executing
@pytest.mark.asyncio
async def test_runner_treats_legacy_entry_point_error_prefix_as_tool_error(tmp_path):
async def test_runner_returns_legacy_entry_point_error_to_model(tmp_path):
provider = MagicMock()
provider.chat_with_retry = AsyncMock(return_value=LLMResponse(
content="working",
tool_calls=[ToolCallRequest(id="call_1", name="legacy_plugin", arguments={})],
usage=None,
))
provider.chat_with_retry = AsyncMock(side_effect=[
LLMResponse(
content="working",
tool_calls=[ToolCallRequest(id="call_1", name="legacy_plugin", arguments={})],
usage=None,
),
LLMResponse(content="reported plugin failure", tool_calls=[], usage=None),
])
result = await AgentRunner().run(make_run_spec(provider,
initial_messages=[{"role": "user", "content": "run plugin"}],
tools=_load_entry_point_plugin(_LegacyErrorPluginTool, tmp_path),
model="test-model",
max_iterations=1,
max_iterations=2,
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
fail_on_tool_error=True,
))
assert result.stop_reason == "tool_error"
assert result.stop_reason == "completed"
assert result.final_content == "reported plugin failure"
assert result.tool_events == [
{"name": "legacy_plugin", "status": "error", "detail": "Error: legacy plugin failed"}
]
@@ -441,7 +444,6 @@ async def test_runner_preserves_structured_plugin_success_that_starts_with_error
model="test-model",
max_iterations=2,
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
fail_on_tool_error=True,
))
assert result.stop_reason == "completed"
tests/agent/test_subagent.py
+21 -23
View File
@@ -12,7 +12,7 @@ from nanobot.agent.tools.filesystem import FileToolsConfig
from nanobot.bus.queue import MessageBus
from nanobot.config.schema import ToolsConfig
from nanobot.llm_usage.context import llm_usage_source
from nanobot.providers.base import GenerationSettings, LLMProvider
from nanobot.providers.base import GenerationSettings, LLMProvider, LLMResponse, ToolCallRequest
from nanobot.security.workspace_access import build_workspace_scope
from nanobot.utils.llm_runtime import LLMRuntime
@@ -168,38 +168,36 @@ async def test_subagent_keeps_project_runtime_scope_with_agent_owned_tools(tmp_p
@pytest.mark.asyncio
async def test_subagent_forwards_fail_on_tool_error_to_runner(tmp_path):
async def test_subagent_recovers_from_tool_error_in_same_run(tmp_path):
provider = MagicMock(spec=LLMProvider)
provider.get_default_model.return_value = "test"
provider.chat_with_retry = AsyncMock(side_effect=[
LLMResponse(
content="reading",
tool_calls=[
ToolCallRequest(
id="call_1",
name="read_file",
arguments={"path": "missing.txt"},
)
],
),
LLMResponse(content="recovered without restarting", tool_calls=[]),
])
sm = SubagentManager(
workspace=tmp_path,
bus=MessageBus(),
max_tool_result_chars=16_000,
fail_on_tool_error=False,
)
sm.runner.run = AsyncMock(
return_value=AgentRunResult(final_content="ok", messages=[], stop_reason="completed")
)
sm._announce_result = AsyncMock()
status = SubagentStatus(
task_id="t1",
label="label",
task_description="task",
started_at=0.0,
)
await sm._run_subagent(
"t1",
"task",
"label",
{"channel": "cli", "chat_id": "direct"},
status,
_runtime(provider),
result = await sm.run_inline(
task="recover after a missing file",
session_key="test:direct",
runtime=_runtime(provider),
)
spec = sm.runner.run.call_args.args[0]
assert spec.fail_on_tool_error is False
assert result == "recovered without restarting"
assert provider.chat_with_retry.await_count == 2
@pytest.mark.asyncio
tests/agent/test_subagent_lifecycle.py
-82
View File
@@ -368,21 +368,6 @@ class TestRunSubagent:
mock_announce.assert_called_once()
assert mock_announce.call_args.args[-2] == "ok"
@pytest.mark.asyncio
async def test_tool_error_run(self, tmp_path):
sm = _manager(tmp_path)
sm.runner.run = AsyncMock(return_value=AgentRunResult(
final_content=None, messages=[], stop_reason="tool_error",
tool_events=[{"name": "read_file", "status": "error", "detail": "not found"}],
))
status = SubagentStatus(task_id="t1", label="label", task_description="do task", started_at=time.monotonic())
with patch.object(sm, "_announce_result", new_callable=AsyncMock) as mock_announce:
await sm._run_subagent(
"t1", "do task", "label",
{"channel": "cli", "chat_id": "direct"}, status, _runtime(),
)
assert mock_announce.call_args.args[-2] == "error"
@pytest.mark.asyncio
async def test_exception_run(self, tmp_path):
sm = _manager(tmp_path)
@@ -504,73 +489,6 @@ class TestAnnounceResult:
assert published[0].metadata["origin_message_id"] == "msg-123"
# ---------------------------------------------------------------------------
# _format_partial_progress
# ---------------------------------------------------------------------------
class TestFormatPartialProgress:
def _make_result(self, tool_events=None, error=None):
return MagicMock(tool_events=tool_events or [], error=error)
def test_completed_only(self):
result = self._make_result(tool_events=[
{"name": "read_file", "status": "ok", "detail": "file content"},
{"name": "exec", "status": "ok", "detail": "output"},
])
text = SubagentManager._format_partial_progress(result)
assert "Completed steps:" in text
assert "read_file" in text
assert "exec" in text
def test_failure_only(self):
result = self._make_result(tool_events=[
{"name": "read_file", "status": "error", "detail": "not found"},
])
text = SubagentManager._format_partial_progress(result)
assert "Failure:" in text
assert "not found" in text
def test_completed_and_failure(self):
result = self._make_result(tool_events=[
{"name": "read_file", "status": "ok", "detail": "content"},
{"name": "exec", "status": "error", "detail": "timeout"},
])
text = SubagentManager._format_partial_progress(result)
assert "Completed steps:" in text
assert "Failure:" in text
def test_limited_to_last_three(self):
result = self._make_result(tool_events=[
{"name": f"tool_{i}", "status": "ok", "detail": f"result_{i}"}
for i in range(5)
])
text = SubagentManager._format_partial_progress(result)
assert "tool_2" in text
assert "tool_3" in text
assert "tool_4" in text
assert "tool_0" not in text
assert "tool_1" not in text
def test_error_without_failure_event(self):
result = self._make_result(
tool_events=[{"name": "read_file", "status": "ok", "detail": "ok"}],
error="Something went wrong",
)
text = SubagentManager._format_partial_progress(result)
assert "Something went wrong" in text
def test_empty_events_with_error(self):
result = self._make_result(error="Total failure")
text = SubagentManager._format_partial_progress(result)
assert "Total failure" in text
def test_empty_no_error_returns_fallback(self):
result = self._make_result()
text = SubagentManager._format_partial_progress(result)
assert "Error" in text
# ---------------------------------------------------------------------------
# cancel_by_session
# ---------------------------------------------------------------------------
tests/agent/test_task_cancel.py
+22 -10
View File
@@ -450,7 +450,9 @@ class TestSubagentCancellation:
mgr._announce_result.assert_awaited_once()
@pytest.mark.asyncio
async def test_subagent_announces_error_when_tool_execution_fails(self, monkeypatch, tmp_path):
async def test_subagent_announces_success_after_recovering_from_tool_failure(
self, monkeypatch, tmp_path
):
from nanobot.agent.subagent import SubagentManager
from nanobot.bus.queue import MessageBus
from nanobot.providers.base import LLMResponse, ToolCallRequest
@@ -458,10 +460,21 @@ class TestSubagentCancellation:
bus = MessageBus()
provider = MagicMock()
provider.get_default_model.return_value = "test-model"
provider.chat_with_retry = AsyncMock(return_value=LLMResponse(
content="thinking",
tool_calls=[ToolCallRequest(id="call_1", name="list_dir", arguments={"path": "."})],
))
provider.chat_with_retry = AsyncMock(side_effect=[
LLMResponse(
content="first attempt",
tool_calls=[
ToolCallRequest(id="call_1", name="list_dir", arguments={"path": "."})
],
),
LLMResponse(
content="retrying",
tool_calls=[
ToolCallRequest(id="call_2", name="list_dir", arguments={"path": "."})
],
),
LLMResponse(content="recovered after tool failure", tool_calls=[]),
])
mgr = SubagentManager(
workspace=tmp_path,
bus=bus,
@@ -492,11 +505,10 @@ class TestSubagentCancellation:
mgr._announce_result.assert_awaited_once()
args = mgr._announce_result.await_args.args
assert "Completed steps:" in args[3]
assert "- list_dir: first result" in args[3]
assert "Failure:" in args[3]
assert "- list_dir: boom" in args[3]
assert args[5] == "error"
assert args[3] == "recovered after tool failure"
assert args[5] == "ok"
assert calls["n"] == 2
assert provider.chat_with_retry.await_count == 3
@pytest.mark.asyncio
async def test_cancel_by_session_cancels_running_subagent_tool(self, monkeypatch, tmp_path):
tests/config/test_config_migration.py
+29
View File
@@ -124,6 +124,35 @@ def test_save_config_drops_legacy_max_messages(tmp_path) -> None:
assert "max_messages" not in saved["agents"]["defaults"]
@pytest.mark.parametrize("field_name", ["failOnToolError", "fail_on_tool_error"])
def test_load_config_ignores_removed_fail_on_tool_error(tmp_path, field_name) -> None:
config_path = tmp_path / "config.json"
config_path.write_text(
json.dumps({"agents": {"defaults": {field_name: True, "maxTokens": 1234}}}),
encoding="utf-8",
)
config = load_config(config_path)
assert config.agents.defaults.max_tokens == 1234
assert not hasattr(config.agents.defaults, "fail_on_tool_error")
def test_save_config_drops_removed_fail_on_tool_error(tmp_path) -> None:
config_path = tmp_path / "config.json"
config_path.write_text(
json.dumps({"agents": {"defaults": {"failOnToolError": True}}}),
encoding="utf-8",
)
config = load_config(config_path)
save_config(config, config_path)
saved = json.loads(config_path.read_text(encoding="utf-8"))
assert "failOnToolError" not in saved["agents"]["defaults"]
assert "fail_on_tool_error" not in saved["agents"]["defaults"]
def test_onboard_refresh_backfills_missing_channel_fields(tmp_path, monkeypatch) -> None:
from nanobot.channels.plugin import load_channel_package