mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-08-06 17:38:35 +00:00
fix(agent): hint repeated tool results
This commit is contained in:
parent
ffdf05a603
commit
e26bb00692
@ -52,6 +52,7 @@ from nanobot.utils.runtime import (
|
|||||||
build_length_recovery_message,
|
build_length_recovery_message,
|
||||||
is_blank_text,
|
is_blank_text,
|
||||||
repeated_external_lookup_error,
|
repeated_external_lookup_error,
|
||||||
|
repeated_tool_result_hint,
|
||||||
repeated_workspace_violation_error,
|
repeated_workspace_violation_error,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -351,6 +352,7 @@ class AgentRunner:
|
|||||||
stop_reason = "completed"
|
stop_reason = "completed"
|
||||||
tool_events: list[dict[str, str]] = []
|
tool_events: list[dict[str, str]] = []
|
||||||
external_lookup_counts: dict[str, int] = {}
|
external_lookup_counts: dict[str, int] = {}
|
||||||
|
repeated_result_counts: dict[str, int] = {}
|
||||||
# Per-turn throttle for repeated attempts against the same outside target.
|
# Per-turn throttle for repeated attempts against the same outside target.
|
||||||
workspace_violation_counts: dict[str, int] = {}
|
workspace_violation_counts: dict[str, int] = {}
|
||||||
empty_content_retries = 0
|
empty_content_retries = 0
|
||||||
@ -468,17 +470,29 @@ class AgentRunner:
|
|||||||
context.tool_results = list(results)
|
context.tool_results = list(results)
|
||||||
context.tool_events = list(new_events)
|
context.tool_events = list(new_events)
|
||||||
completed_tool_results: list[dict[str, Any]] = []
|
completed_tool_results: list[dict[str, Any]] = []
|
||||||
for tool_call, result in zip(response.tool_calls, results):
|
for tool_call, result, event in zip(response.tool_calls, results, new_events):
|
||||||
|
content = self.context_governor.normalize_tool_result(
|
||||||
|
governance_config,
|
||||||
|
tool_call.id,
|
||||||
|
tool_call.name,
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
if event.get("status") == "ok":
|
||||||
|
result_hint = repeated_tool_result_hint(
|
||||||
|
tool_call.name,
|
||||||
|
content,
|
||||||
|
repeated_result_counts,
|
||||||
|
)
|
||||||
|
if result_hint:
|
||||||
|
if isinstance(content, str):
|
||||||
|
content = content + result_hint
|
||||||
|
elif isinstance(content, list):
|
||||||
|
content = [*content, {"type": "text", "text": result_hint.strip()}]
|
||||||
tool_message = {
|
tool_message = {
|
||||||
"role": "tool",
|
"role": "tool",
|
||||||
"tool_call_id": tool_call.id,
|
"tool_call_id": tool_call.id,
|
||||||
"name": tool_call.name,
|
"name": tool_call.name,
|
||||||
"content": self.context_governor.normalize_tool_result(
|
"content": content,
|
||||||
governance_config,
|
|
||||||
tool_call.id,
|
|
||||||
tool_call.name,
|
|
||||||
result,
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
messages.append(tool_message)
|
messages.append(tool_message)
|
||||||
completed_tool_results.append(tool_message)
|
completed_tool_results.append(tool_message)
|
||||||
@ -1135,7 +1149,10 @@ class AgentRunner:
|
|||||||
if spec.concurrent_tools and len(batch) > 1:
|
if spec.concurrent_tools and len(batch) > 1:
|
||||||
batch_results = await asyncio.gather(*(
|
batch_results = await asyncio.gather(*(
|
||||||
self._run_tool(
|
self._run_tool(
|
||||||
spec, tool_call, external_lookup_counts, workspace_violation_counts,
|
spec,
|
||||||
|
tool_call,
|
||||||
|
external_lookup_counts,
|
||||||
|
workspace_violation_counts,
|
||||||
)
|
)
|
||||||
for tool_call in batch
|
for tool_call in batch
|
||||||
))
|
))
|
||||||
@ -1144,7 +1161,10 @@ class AgentRunner:
|
|||||||
batch_results = []
|
batch_results = []
|
||||||
for tool_call in batch:
|
for tool_call in batch:
|
||||||
result = await self._run_tool(
|
result = await self._run_tool(
|
||||||
spec, tool_call, external_lookup_counts, workspace_violation_counts,
|
spec,
|
||||||
|
tool_call,
|
||||||
|
external_lookup_counts,
|
||||||
|
workspace_violation_counts,
|
||||||
)
|
)
|
||||||
tool_results.append(result)
|
tool_results.append(result)
|
||||||
batch_results.append(result)
|
batch_results.append(result)
|
||||||
|
|||||||
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import re
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@ -10,7 +11,7 @@ from loguru import logger
|
|||||||
|
|
||||||
from nanobot.utils.helpers import stringify_text_blocks
|
from nanobot.utils.helpers import stringify_text_blocks
|
||||||
|
|
||||||
_MAX_REPEAT_EXTERNAL_LOOKUPS = 2
|
_MAX_REPEAT_ATTEMPTS = 2
|
||||||
|
|
||||||
# Third same-target workspace violation in a turn escalates to "stop retrying".
|
# Third same-target workspace violation in a turn escalates to "stop retrying".
|
||||||
_MAX_REPEAT_WORKSPACE_VIOLATIONS = 2
|
_MAX_REPEAT_WORKSPACE_VIOLATIONS = 2
|
||||||
@ -103,6 +104,14 @@ def external_lookup_signature(tool_name: str, arguments: Any) -> str | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _over_repeat_budget(signature: str | None, seen_counts: dict[str, int]) -> int | None:
|
||||||
|
if signature is None:
|
||||||
|
return None
|
||||||
|
count = seen_counts.get(signature, 0) + 1
|
||||||
|
seen_counts[signature] = count
|
||||||
|
return count if count > _MAX_REPEAT_ATTEMPTS else None
|
||||||
|
|
||||||
|
|
||||||
def repeated_external_lookup_error(
|
def repeated_external_lookup_error(
|
||||||
tool_name: str,
|
tool_name: str,
|
||||||
arguments: Any,
|
arguments: Any,
|
||||||
@ -110,11 +119,8 @@ def repeated_external_lookup_error(
|
|||||||
) -> str | None:
|
) -> str | None:
|
||||||
"""Block repeated external lookups after a small retry budget."""
|
"""Block repeated external lookups after a small retry budget."""
|
||||||
signature = external_lookup_signature(tool_name, arguments)
|
signature = external_lookup_signature(tool_name, arguments)
|
||||||
if signature is None:
|
count = _over_repeat_budget(signature, seen_counts)
|
||||||
return None
|
if count is None:
|
||||||
count = seen_counts.get(signature, 0) + 1
|
|
||||||
seen_counts[signature] = count
|
|
||||||
if count <= _MAX_REPEAT_EXTERNAL_LOOKUPS:
|
|
||||||
return None
|
return None
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Blocking repeated external lookup {} on attempt {}",
|
"Blocking repeated external lookup {} on attempt {}",
|
||||||
@ -127,6 +133,33 @@ def repeated_external_lookup_error(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def repeated_tool_result_hint(
|
||||||
|
tool_name: str,
|
||||||
|
result: Any,
|
||||||
|
seen_counts: dict[str, int],
|
||||||
|
) -> str | None:
|
||||||
|
"""Hint when a successful tool keeps returning the exact same text in one turn."""
|
||||||
|
if isinstance(result, str):
|
||||||
|
text = result
|
||||||
|
elif isinstance(result, list):
|
||||||
|
text = stringify_text_blocks(result)
|
||||||
|
else:
|
||||||
|
text = None
|
||||||
|
if text is None:
|
||||||
|
return None
|
||||||
|
digest = hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()
|
||||||
|
signature = f"tool_result:{tool_name}:{len(text)}:{digest}"
|
||||||
|
count = _over_repeat_budget(signature, seen_counts)
|
||||||
|
if count is None:
|
||||||
|
return None
|
||||||
|
logger.warning("Hinting repeated {} result on attempt {}", tool_name, count)
|
||||||
|
return (
|
||||||
|
f"\n\n[Repeated {tool_name} result: this exact output has already been "
|
||||||
|
"returned in this turn. Use the existing evidence, or change the tool input "
|
||||||
|
"if you need new information.]"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# Workspace-boundary violations are soft errors, with per-target throttling.
|
# Workspace-boundary violations are soft errors, with per-target throttling.
|
||||||
|
|
||||||
_OUTSIDE_PATH_PATTERN = re.compile(r"(?:^|[\s|>'\"])((?:/[^\s\"'>;|<]+)|(?:~[^\s\"'>;|<]+))")
|
_OUTSIDE_PATH_PATTERN = re.compile(r"(?:^|[\s|>'\"])((?:/[^\s\"'>;|<]+)|(?:~[^\s\"'>;|<]+))")
|
||||||
|
|||||||
@ -465,3 +465,83 @@ async def test_runner_blocks_repeated_external_fetches():
|
|||||||
if msg.get("role") == "tool" and msg.get("tool_call_id") == "call_3"
|
if msg.get("role") == "tool" and msg.get("tool_call_id") == "call_3"
|
||||||
][0]
|
][0]
|
||||||
assert "repeated external lookup blocked" in blocked_tool_message["content"]
|
assert "repeated external lookup blocked" in blocked_tool_message["content"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runner_hints_repeated_tool_results():
|
||||||
|
provider = MagicMock()
|
||||||
|
captured_final_call: list[dict] = []
|
||||||
|
call_count = {"n": 0}
|
||||||
|
|
||||||
|
async def chat_with_retry(*, messages, **kwargs):
|
||||||
|
call_count["n"] += 1
|
||||||
|
if call_count["n"] <= 3:
|
||||||
|
return LLMResponse(
|
||||||
|
content="reading",
|
||||||
|
tool_calls=[ToolCallRequest(
|
||||||
|
id=f"call_{call_count['n']}",
|
||||||
|
name="grep",
|
||||||
|
arguments={"pattern": "TODO", "path": "nanobot"},
|
||||||
|
)],
|
||||||
|
usage={},
|
||||||
|
)
|
||||||
|
captured_final_call[:] = messages
|
||||||
|
return LLMResponse(content="done", tool_calls=[], usage={})
|
||||||
|
|
||||||
|
provider.chat_with_retry = chat_with_retry
|
||||||
|
tools = MagicMock()
|
||||||
|
tools.get_definitions.return_value = []
|
||||||
|
tools.execute = AsyncMock(return_value="file content")
|
||||||
|
|
||||||
|
result = await AgentRunner(provider).run(AgentRunSpec(
|
||||||
|
initial_messages=[{"role": "user", "content": "review code"}],
|
||||||
|
tools=tools,
|
||||||
|
model="test-model",
|
||||||
|
max_iterations=4,
|
||||||
|
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
|
||||||
|
))
|
||||||
|
|
||||||
|
assert result.final_content == "done"
|
||||||
|
assert tools.execute.await_count == 3
|
||||||
|
hinted_tool_message = [
|
||||||
|
msg for msg in captured_final_call
|
||||||
|
if msg.get("role") == "tool" and msg.get("tool_call_id") == "call_3"
|
||||||
|
][0]
|
||||||
|
assert "Repeated grep result" in hinted_tool_message["content"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runner_does_not_hint_different_tool_results():
|
||||||
|
provider = MagicMock()
|
||||||
|
call_count = {"n": 0}
|
||||||
|
|
||||||
|
async def chat_with_retry(*, messages, **kwargs):
|
||||||
|
call_count["n"] += 1
|
||||||
|
if call_count["n"] <= 3:
|
||||||
|
return LLMResponse(
|
||||||
|
content="reading",
|
||||||
|
tool_calls=[ToolCallRequest(
|
||||||
|
id=f"call_{call_count['n']}",
|
||||||
|
name="grep",
|
||||||
|
arguments={"pattern": "TODO", "path": "nanobot"},
|
||||||
|
)],
|
||||||
|
usage={},
|
||||||
|
)
|
||||||
|
return LLMResponse(content="done", tool_calls=[], usage={})
|
||||||
|
|
||||||
|
provider.chat_with_retry = chat_with_retry
|
||||||
|
tools = MagicMock()
|
||||||
|
tools.get_definitions.return_value = []
|
||||||
|
tools.execute = AsyncMock(side_effect=["first result", "second result", "third result"])
|
||||||
|
|
||||||
|
result = await AgentRunner(provider).run(AgentRunSpec(
|
||||||
|
initial_messages=[{"role": "user", "content": "review code"}],
|
||||||
|
tools=tools,
|
||||||
|
model="test-model",
|
||||||
|
max_iterations=4,
|
||||||
|
max_tool_result_chars=_MAX_TOOL_RESULT_CHARS,
|
||||||
|
))
|
||||||
|
|
||||||
|
assert result.final_content == "done"
|
||||||
|
assert tools.execute.await_count == 3
|
||||||
|
assert all("Repeated grep result" not in str(msg.get("content", "")) for msg in result.messages)
|
||||||
|
|||||||
60
tests/utils/test_repeated_tool_result_hint.py
Normal file
60
tests/utils/test_repeated_tool_result_hint.py
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
"""Tests for repeated tool-result hints."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from nanobot.utils.runtime import (
|
||||||
|
repeated_external_lookup_error,
|
||||||
|
repeated_tool_result_hint,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_repeated_tool_result_hints_after_two_identical_results():
|
||||||
|
counts: dict[str, int] = {}
|
||||||
|
|
||||||
|
assert repeated_tool_result_hint("grep", "same result", counts) is None
|
||||||
|
assert repeated_tool_result_hint("grep", "same result", counts) is None
|
||||||
|
third = repeated_tool_result_hint("grep", "same result", counts)
|
||||||
|
|
||||||
|
assert third is not None
|
||||||
|
assert "Repeated grep result" in third
|
||||||
|
|
||||||
|
|
||||||
|
def test_repeated_tool_result_ignores_different_results():
|
||||||
|
counts: dict[str, int] = {}
|
||||||
|
|
||||||
|
assert repeated_tool_result_hint("grep", "first", counts) is None
|
||||||
|
assert repeated_tool_result_hint("grep", "second", counts) is None
|
||||||
|
assert repeated_tool_result_hint("grep", "third", counts) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_repeated_tool_result_is_per_tool():
|
||||||
|
counts: dict[str, int] = {}
|
||||||
|
|
||||||
|
repeated_tool_result_hint("grep", "same", counts)
|
||||||
|
repeated_tool_result_hint("grep", "same", counts)
|
||||||
|
|
||||||
|
assert repeated_tool_result_hint("read_file", "same", counts) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_repeated_tool_result_handles_text_blocks():
|
||||||
|
counts: dict[str, int] = {}
|
||||||
|
result = [{"type": "text", "text": "same result"}]
|
||||||
|
|
||||||
|
repeated_tool_result_hint("mcp", result, counts)
|
||||||
|
repeated_tool_result_hint("mcp", result, counts)
|
||||||
|
third = repeated_tool_result_hint("mcp", result, counts)
|
||||||
|
|
||||||
|
assert third is not None
|
||||||
|
assert "Repeated mcp result" in third
|
||||||
|
|
||||||
|
|
||||||
|
def test_repeated_external_lookup_still_blocks_after_two_attempts():
|
||||||
|
counts: dict[str, int] = {}
|
||||||
|
arguments = {"url": "https://example.com"}
|
||||||
|
|
||||||
|
repeated_external_lookup_error("web_fetch", arguments, counts)
|
||||||
|
repeated_external_lookup_error("web_fetch", arguments, counts)
|
||||||
|
third = repeated_external_lookup_error("web_fetch", arguments, counts)
|
||||||
|
|
||||||
|
assert third is not None
|
||||||
|
assert "repeated external lookup blocked" in third
|
||||||
Loading…
x
Reference in New Issue
Block a user