fix(web): fall back when readability is unavailable

2026-08-07 21:08:34 +03:00 · 2026-06-06 00:49:35 +08:00
parent 5606653f47
commit 7449e0a770
3 changed files with 62 additions and 7 deletions
@@ -826,12 +826,12 @@ class WebFetchTool(Tool):
            if "application/json" in ctype:
                text, extractor = json.dumps(r.json(), indent=2, ensure_ascii=False), "json"
            elif "text/html" in ctype or r.text[:256].lower().startswith(("<!doctype", "<html")):
-                from readability import Document
-
-                doc = Document(r.text)
-                content = self._to_markdown(doc.summary()) if extract_mode == "markdown" else _strip_tags(doc.summary())
-                text = f"# {doc.title()}\n\n{content}" if doc.title() else content
-                extractor = "readability"
+                try:
+                    text = self._extract_readable_html(r.text, extract_mode)
+                    extractor = "readability"
+                except Exception as e:
+                    logger.warning("Readability failed for {}, using raw HTML fallback: {}", url, e)
+                    text, extractor = _normalize(_strip_tags(r.text)), "html"
            else:
                text, extractor = r.text, "raw"

@@ -852,6 +852,14 @@ class WebFetchTool(Tool):
            logger.exception("WebFetch error for {}", url)
            return json.dumps({"error": str(e), "url": url}, ensure_ascii=False)

+    def _extract_readable_html(self, html_content: str, extract_mode: str) -> str:
+        from readability import Document
+
+        doc = Document(html_content)
+        summary = doc.summary()
+        content = self._to_markdown(summary) if extract_mode == "markdown" else _strip_tags(summary)
+        return f"# {doc.title()}\n\n{content}" if doc.title() else content
+
    def _to_markdown(self, html_content: str) -> str:
        """Convert HTML to markdown."""
        text = re.sub(r'<a\s+[^>]*href=["\']([^"\']+)["\'][^>]*>([\s\S]*?)</a>',
@@ -34,6 +34,7 @@ dependencies = [
    "oauth-cli-kit>=0.1.3,<1.0.0",
    "loguru>=0.7.3,<1.0.0",
    "readability-lxml>=0.8.4,<1.0.0",
+    "lxml-html-clean>=0.4.0,<1.0.0",
    "rich>=14.0.0,<15.0.0",
    "croniter>=6.0.0,<7.0.0",
    "dingtalk-stream>=0.24.0,<1.0.0",
@@ -12,7 +12,11 @@ import pytest
 from nanobot.agent.tools import web as web_module
 from nanobot.agent.tools.web import WebFetchTool
 from nanobot.config.schema import WebFetchConfig
-from nanobot.security.workspace_access import bind_workspace_scope, build_workspace_scope, reset_workspace_scope
+from nanobot.security.workspace_access import (
+    bind_workspace_scope,
+    build_workspace_scope,
+    reset_workspace_scope,
+)

 _REAL_GETADDRINFO = socket.getaddrinfo

@@ -147,6 +151,7 @@ async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):
            return FakeResponse()

    monkeypatch.setattr(tool, "_fetch_jina", _fail_jina)
+    monkeypatch.setattr(tool, "_extract_readable_html", lambda html, mode: "Hello world")
    monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient)

    with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
@@ -160,6 +165,47 @@ async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):
    ]


+@pytest.mark.asyncio
+async def test_web_fetch_falls_back_when_readability_dependency_is_missing(monkeypatch):
+    tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))
+
+    class FakeResponse:
+        status_code = 200
+        url = "https://example.com/page"
+        text = "<html><head><title>Test</title></head><body><p>Hello world</p></body></html>"
+        headers = {"content-type": "text/html"}
+
+        def raise_for_status(self):
+            return None
+
+    class FakeClient:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, exc_type, exc, tb):
+            return False
+
+        async def get(self, url, headers=None, follow_redirects=False, **kwargs):
+            return FakeResponse()
+
+    def _missing_readability(*args, **kwargs):
+        raise ModuleNotFoundError("No module named 'lxml_html_clean'")
+
+    monkeypatch.setattr(tool, "_extract_readable_html", _missing_readability)
+    monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient)
+
+    with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
+        result = await tool._fetch_readability("https://example.com/page", "markdown", 5000)
+
+    data = json.loads(result)
+    assert data["extractor"] == "html"
+    assert data["untrusted"] is True
+    assert "Hello world" in data["text"]
+
+
@pytest.mark.asyncio
 async def test_web_fetch_blocks_private_redirect_before_readability_request(monkeypatch):
    tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))