mirror/nanobot
1
0
Fork 0
mirror of https://github.com/HKUDS/nanobot.git synced 2026-08-08 05:18:49 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(web): fall back when readability is unavailable

Browse Source
This commit is contained in:
Xubin Ren
2026-06-06 00:49:35 +08:00
parent 5606653f47
commit 7449e0a770
3 changed files with 62 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nanobot/agent/tools/web.py
+14 -6
View File
@@ -826,12 +826,12 @@ class WebFetchTool(Tool):
if "application/json" in ctype: if "application/json" in ctype:
text, extractor = json.dumps(r.json(), indent=2, ensure_ascii=False), "json" text, extractor = json.dumps(r.json(), indent=2, ensure_ascii=False), "json"
elif "text/html" in ctype or r.text[:256].lower().startswith(("<!doctype", "<html")): elif "text/html" in ctype or r.text[:256].lower().startswith(("<!doctype", "<html")):
from readability import Document try:
text = self._extract_readable_html(r.text, extract_mode)
doc = Document(r.text) extractor = "readability"
content = self._to_markdown(doc.summary()) if extract_mode == "markdown" else _strip_tags(doc.summary()) except Exception as e:
text = f"# {doc.title()}\n\n{content}" if doc.title() else content logger.warning("Readability failed for {}, using raw HTML fallback: {}", url, e)
extractor = "readability" text, extractor = _normalize(_strip_tags(r.text)), "html"
else: else:
text, extractor = r.text, "raw" text, extractor = r.text, "raw"
@@ -852,6 +852,14 @@ class WebFetchTool(Tool):
logger.exception("WebFetch error for {}", url) logger.exception("WebFetch error for {}", url)
return json.dumps({"error": str(e), "url": url}, ensure_ascii=False) return json.dumps({"error": str(e), "url": url}, ensure_ascii=False)
def _extract_readable_html(self, html_content: str, extract_mode: str) -> str:
from readability import Document
doc = Document(html_content)
summary = doc.summary()
content = self._to_markdown(summary) if extract_mode == "markdown" else _strip_tags(summary)
return f"# {doc.title()}\n\n{content}" if doc.title() else content
def _to_markdown(self, html_content: str) -> str: def _to_markdown(self, html_content: str) -> str:
"""Convert HTML to markdown.""" """Convert HTML to markdown."""
text = re.sub(r'<a\s+[^>]*href=["\']([^"\']+)["\'][^>]*>([\s\S]*?)</a>', text = re.sub(r'<a\s+[^>]*href=["\']([^"\']+)["\'][^>]*>([\s\S]*?)</a>',
pyproject.toml
+1
View File
@@ -34,6 +34,7 @@ dependencies = [
"oauth-cli-kit>=0.1.3,<1.0.0", "oauth-cli-kit>=0.1.3,<1.0.0",
"loguru>=0.7.3,<1.0.0", "loguru>=0.7.3,<1.0.0",
"readability-lxml>=0.8.4,<1.0.0", "readability-lxml>=0.8.4,<1.0.0",
"lxml-html-clean>=0.4.0,<1.0.0",
"rich>=14.0.0,<15.0.0", "rich>=14.0.0,<15.0.0",
"croniter>=6.0.0,<7.0.0", "croniter>=6.0.0,<7.0.0",
"dingtalk-stream>=0.24.0,<1.0.0", "dingtalk-stream>=0.24.0,<1.0.0",
tests/tools/test_web_fetch_security.py
+47 -1
View File
@@ -12,7 +12,11 @@ import pytest
from nanobot.agent.tools import web as web_module from nanobot.agent.tools import web as web_module
from nanobot.agent.tools.web import WebFetchTool from nanobot.agent.tools.web import WebFetchTool
from nanobot.config.schema import WebFetchConfig from nanobot.config.schema import WebFetchConfig
from nanobot.security.workspace_access import bind_workspace_scope, build_workspace_scope, reset_workspace_scope from nanobot.security.workspace_access import (
bind_workspace_scope,
build_workspace_scope,
reset_workspace_scope,
)
_REAL_GETADDRINFO = socket.getaddrinfo _REAL_GETADDRINFO = socket.getaddrinfo
@@ -147,6 +151,7 @@ async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):
return FakeResponse() return FakeResponse()
monkeypatch.setattr(tool, "_fetch_jina", _fail_jina) monkeypatch.setattr(tool, "_fetch_jina", _fail_jina)
monkeypatch.setattr(tool, "_extract_readable_html", lambda html, mode: "Hello world")
monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient) monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient)
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public): with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
@@ -160,6 +165,47 @@ async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):
] ]
@pytest.mark.asyncio
async def test_web_fetch_falls_back_when_readability_dependency_is_missing(monkeypatch):
tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))
class FakeResponse:
status_code = 200
url = "https://example.com/page"
text = "<html><head><title>Test</title></head><body><p>Hello world</p></body></html>"
headers = {"content-type": "text/html"}
def raise_for_status(self):
return None
class FakeClient:
def __init__(self, *args, **kwargs):
pass
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
async def get(self, url, headers=None, follow_redirects=False, **kwargs):
return FakeResponse()
def _missing_readability(*args, **kwargs):
raise ModuleNotFoundError("No module named 'lxml_html_clean'")
monkeypatch.setattr(tool, "_extract_readable_html", _missing_readability)
monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient)
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
result = await tool._fetch_readability("https://example.com/page", "markdown", 5000)
data = json.loads(result)
assert data["extractor"] == "html"
assert data["untrusted"] is True
assert "Hello world" in data["text"]
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_web_fetch_blocks_private_redirect_before_readability_request(monkeypatch): async def test_web_fetch_blocks_private_redirect_before_readability_request(monkeypatch):
tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False)) tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))