diff --git a/nanobot/agent/runner.py b/nanobot/agent/runner.py index 645e1f482..92b9592d3 100644 --- a/nanobot/agent/runner.py +++ b/nanobot/agent/runner.py @@ -52,6 +52,7 @@ from nanobot.utils.runtime import ( build_length_recovery_message, is_blank_text, repeated_external_lookup_error, + repeated_tool_result_hint, repeated_workspace_violation_error, ) @@ -351,6 +352,7 @@ class AgentRunner: stop_reason = "completed" tool_events: list[dict[str, str]] = [] external_lookup_counts: dict[str, int] = {} + repeated_result_counts: dict[str, int] = {} # Per-turn throttle for repeated attempts against the same outside target. workspace_violation_counts: dict[str, int] = {} empty_content_retries = 0 @@ -468,17 +470,29 @@ class AgentRunner: context.tool_results = list(results) context.tool_events = list(new_events) completed_tool_results: list[dict[str, Any]] = [] - for tool_call, result in zip(response.tool_calls, results): + for tool_call, result, event in zip(response.tool_calls, results, new_events): + content = self.context_governor.normalize_tool_result( + governance_config, + tool_call.id, + tool_call.name, + result, + ) + if event.get("status") == "ok": + result_hint = repeated_tool_result_hint( + tool_call.name, + content, + repeated_result_counts, + ) + if result_hint: + if isinstance(content, str): + content = content + result_hint + elif isinstance(content, list): + content = [*content, {"type": "text", "text": result_hint.strip()}] tool_message = { "role": "tool", "tool_call_id": tool_call.id, "name": tool_call.name, - "content": self.context_governor.normalize_tool_result( - governance_config, - tool_call.id, - tool_call.name, - result, - ), + "content": content, } messages.append(tool_message) completed_tool_results.append(tool_message) @@ -1135,7 +1149,10 @@ class AgentRunner: if spec.concurrent_tools and len(batch) > 1: batch_results = await asyncio.gather(*( self._run_tool( - spec, tool_call, external_lookup_counts, workspace_violation_counts, + spec, + tool_call, + external_lookup_counts, + workspace_violation_counts, ) for tool_call in batch )) @@ -1144,7 +1161,10 @@ class AgentRunner: batch_results = [] for tool_call in batch: result = await self._run_tool( - spec, tool_call, external_lookup_counts, workspace_violation_counts, + spec, + tool_call, + external_lookup_counts, + workspace_violation_counts, ) tool_results.append(result) batch_results.append(result) diff --git a/nanobot/utils/runtime.py b/nanobot/utils/runtime.py index 9141583ea..7c489e0e0 100644 --- a/nanobot/utils/runtime.py +++ b/nanobot/utils/runtime.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import re from pathlib import Path from typing import Any @@ -10,7 +11,7 @@ from loguru import logger from nanobot.utils.helpers import stringify_text_blocks -_MAX_REPEAT_EXTERNAL_LOOKUPS = 2 +_MAX_REPEAT_ATTEMPTS = 2 # Third same-target workspace violation in a turn escalates to "stop retrying". _MAX_REPEAT_WORKSPACE_VIOLATIONS = 2 @@ -103,6 +104,14 @@ def external_lookup_signature(tool_name: str, arguments: Any) -> str | None: return None +def _over_repeat_budget(signature: str | None, seen_counts: dict[str, int]) -> int | None: + if signature is None: + return None + count = seen_counts.get(signature, 0) + 1 + seen_counts[signature] = count + return count if count > _MAX_REPEAT_ATTEMPTS else None + + def repeated_external_lookup_error( tool_name: str, arguments: Any, @@ -110,11 +119,8 @@ def repeated_external_lookup_error( ) -> str | None: """Block repeated external lookups after a small retry budget.""" signature = external_lookup_signature(tool_name, arguments) - if signature is None: - return None - count = seen_counts.get(signature, 0) + 1 - seen_counts[signature] = count - if count <= _MAX_REPEAT_EXTERNAL_LOOKUPS: + count = _over_repeat_budget(signature, seen_counts) + if count is None: return None logger.warning( "Blocking repeated external lookup {} on attempt {}", @@ -127,6 +133,33 @@ def repeated_external_lookup_error( ) +def repeated_tool_result_hint( + tool_name: str, + result: Any, + seen_counts: dict[str, int], +) -> str | None: + """Hint when a successful tool keeps returning the exact same text in one turn.""" + if isinstance(result, str): + text = result + elif isinstance(result, list): + text = stringify_text_blocks(result) + else: + text = None + if text is None: + return None + digest = hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest() + signature = f"tool_result:{tool_name}:{len(text)}:{digest}" + count = _over_repeat_budget(signature, seen_counts) + if count is None: + return None + logger.warning("Hinting repeated {} result on attempt {}", tool_name, count) + return ( + f"\n\n[Repeated {tool_name} result: this exact output has already been " + "returned in this turn. Use the existing evidence, or change the tool input " + "if you need new information.]" + ) + + # Workspace-boundary violations are soft errors, with per-target throttling. _OUTSIDE_PATH_PATTERN = re.compile(r"(?:^|[\s|>'\"])((?:/[^\s\"'>;|<]+)|(?:~[^\s\"'>;|<]+))") diff --git a/tests/agent/test_runner_tool_execution.py b/tests/agent/test_runner_tool_execution.py index 328bf8dd4..f6e3a30a7 100644 --- a/tests/agent/test_runner_tool_execution.py +++ b/tests/agent/test_runner_tool_execution.py @@ -465,3 +465,83 @@ async def test_runner_blocks_repeated_external_fetches(): if msg.get("role") == "tool" and msg.get("tool_call_id") == "call_3" ][0] assert "repeated external lookup blocked" in blocked_tool_message["content"] + + +@pytest.mark.asyncio +async def test_runner_hints_repeated_tool_results(): + provider = MagicMock() + captured_final_call: list[dict] = [] + call_count = {"n": 0} + + async def chat_with_retry(*, messages, **kwargs): + call_count["n"] += 1 + if call_count["n"] <= 3: + return LLMResponse( + content="reading", + tool_calls=[ToolCallRequest( + id=f"call_{call_count['n']}", + name="grep", + arguments={"pattern": "TODO", "path": "nanobot"}, + )], + usage={}, + ) + captured_final_call[:] = messages + return LLMResponse(content="done", tool_calls=[], usage={}) + + provider.chat_with_retry = chat_with_retry + tools = MagicMock() + tools.get_definitions.return_value = [] + tools.execute = AsyncMock(return_value="file content") + + result = await AgentRunner(provider).run(AgentRunSpec( + initial_messages=[{"role": "user", "content": "review code"}], + tools=tools, + model="test-model", + max_iterations=4, + max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, + )) + + assert result.final_content == "done" + assert tools.execute.await_count == 3 + hinted_tool_message = [ + msg for msg in captured_final_call + if msg.get("role") == "tool" and msg.get("tool_call_id") == "call_3" + ][0] + assert "Repeated grep result" in hinted_tool_message["content"] + + +@pytest.mark.asyncio +async def test_runner_does_not_hint_different_tool_results(): + provider = MagicMock() + call_count = {"n": 0} + + async def chat_with_retry(*, messages, **kwargs): + call_count["n"] += 1 + if call_count["n"] <= 3: + return LLMResponse( + content="reading", + tool_calls=[ToolCallRequest( + id=f"call_{call_count['n']}", + name="grep", + arguments={"pattern": "TODO", "path": "nanobot"}, + )], + usage={}, + ) + return LLMResponse(content="done", tool_calls=[], usage={}) + + provider.chat_with_retry = chat_with_retry + tools = MagicMock() + tools.get_definitions.return_value = [] + tools.execute = AsyncMock(side_effect=["first result", "second result", "third result"]) + + result = await AgentRunner(provider).run(AgentRunSpec( + initial_messages=[{"role": "user", "content": "review code"}], + tools=tools, + model="test-model", + max_iterations=4, + max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, + )) + + assert result.final_content == "done" + assert tools.execute.await_count == 3 + assert all("Repeated grep result" not in str(msg.get("content", "")) for msg in result.messages) diff --git a/tests/utils/test_repeated_tool_result_hint.py b/tests/utils/test_repeated_tool_result_hint.py new file mode 100644 index 000000000..cc276cde3 --- /dev/null +++ b/tests/utils/test_repeated_tool_result_hint.py @@ -0,0 +1,60 @@ +"""Tests for repeated tool-result hints.""" + +from __future__ import annotations + +from nanobot.utils.runtime import ( + repeated_external_lookup_error, + repeated_tool_result_hint, +) + + +def test_repeated_tool_result_hints_after_two_identical_results(): + counts: dict[str, int] = {} + + assert repeated_tool_result_hint("grep", "same result", counts) is None + assert repeated_tool_result_hint("grep", "same result", counts) is None + third = repeated_tool_result_hint("grep", "same result", counts) + + assert third is not None + assert "Repeated grep result" in third + + +def test_repeated_tool_result_ignores_different_results(): + counts: dict[str, int] = {} + + assert repeated_tool_result_hint("grep", "first", counts) is None + assert repeated_tool_result_hint("grep", "second", counts) is None + assert repeated_tool_result_hint("grep", "third", counts) is None + + +def test_repeated_tool_result_is_per_tool(): + counts: dict[str, int] = {} + + repeated_tool_result_hint("grep", "same", counts) + repeated_tool_result_hint("grep", "same", counts) + + assert repeated_tool_result_hint("read_file", "same", counts) is None + + +def test_repeated_tool_result_handles_text_blocks(): + counts: dict[str, int] = {} + result = [{"type": "text", "text": "same result"}] + + repeated_tool_result_hint("mcp", result, counts) + repeated_tool_result_hint("mcp", result, counts) + third = repeated_tool_result_hint("mcp", result, counts) + + assert third is not None + assert "Repeated mcp result" in third + + +def test_repeated_external_lookup_still_blocks_after_two_attempts(): + counts: dict[str, int] = {} + arguments = {"url": "https://example.com"} + + repeated_external_lookup_error("web_fetch", arguments, counts) + repeated_external_lookup_error("web_fetch", arguments, counts) + third = repeated_external_lookup_error("web_fetch", arguments, counts) + + assert third is not None + assert "repeated external lookup blocked" in third