From 7449e0a770a60cd4d0c23053784b3b7b987fee3a Mon Sep 17 00:00:00 2001
From: Xubin Ren <52506698+Re-bin@users.noreply.github.com>
Date: Sat, 6 Jun 2026 00:49:35 +0800
Subject: [PATCH] fix(web): fall back when readability is unavailable

---
 nanobot/agent/tools/web.py             | 20 +++++++----
 pyproject.toml                         |  1 +
 tests/tools/test_web_fetch_security.py | 48 +++++++++++++++++++++++++-
 3 files changed, 62 insertions(+), 7 deletions(-)

diff --git a/nanobot/agent/tools/web.py b/nanobot/agent/tools/web.py
index 4c202eaee..f4221ca5b 100644
--- a/nanobot/agent/tools/web.py
+++ b/nanobot/agent/tools/web.py
@@ -826,12 +826,12 @@ class WebFetchTool(Tool):
             if "application/json" in ctype:
                 text, extractor = json.dumps(r.json(), indent=2, ensure_ascii=False), "json"
             elif "text/html" in ctype or r.text[:256].lower().startswith(("<!doctype", "<html")):
-                from readability import Document
-
-                doc = Document(r.text)
-                content = self._to_markdown(doc.summary()) if extract_mode == "markdown" else _strip_tags(doc.summary())
-                text = f"# {doc.title()}\n\n{content}" if doc.title() else content
-                extractor = "readability"
+                try:
+                    text = self._extract_readable_html(r.text, extract_mode)
+                    extractor = "readability"
+                except Exception as e:
+                    logger.warning("Readability failed for {}, using raw HTML fallback: {}", url, e)
+                    text, extractor = _normalize(_strip_tags(r.text)), "html"
             else:
                 text, extractor = r.text, "raw"
 
@@ -852,6 +852,14 @@ class WebFetchTool(Tool):
             logger.exception("WebFetch error for {}", url)
             return json.dumps({"error": str(e), "url": url}, ensure_ascii=False)
 
+    def _extract_readable_html(self, html_content: str, extract_mode: str) -> str:
+        from readability import Document
+
+        doc = Document(html_content)
+        summary = doc.summary()
+        content = self._to_markdown(summary) if extract_mode == "markdown" else _strip_tags(summary)
+        return f"# {doc.title()}\n\n{content}" if doc.title() else content
+
     def _to_markdown(self, html_content: str) -> str:
         """Convert HTML to markdown."""
         text = re.sub(r'<a\s+[^>]*href=["\']([^"\']+)["\'][^>]*>([\s\S]*?)</a>',
diff --git a/pyproject.toml b/pyproject.toml
index 7adbb9c51..915bd8c3d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,6 +34,7 @@ dependencies = [
     "oauth-cli-kit>=0.1.3,<1.0.0",
     "loguru>=0.7.3,<1.0.0",
     "readability-lxml>=0.8.4,<1.0.0",
+    "lxml-html-clean>=0.4.0,<1.0.0",
     "rich>=14.0.0,<15.0.0",
     "croniter>=6.0.0,<7.0.0",
     "dingtalk-stream>=0.24.0,<1.0.0",
diff --git a/tests/tools/test_web_fetch_security.py b/tests/tools/test_web_fetch_security.py
index 89ff9d9f9..6fb1d0f64 100644
--- a/tests/tools/test_web_fetch_security.py
+++ b/tests/tools/test_web_fetch_security.py
@@ -12,7 +12,11 @@ import pytest
 from nanobot.agent.tools import web as web_module
 from nanobot.agent.tools.web import WebFetchTool
 from nanobot.config.schema import WebFetchConfig
-from nanobot.security.workspace_access import bind_workspace_scope, build_workspace_scope, reset_workspace_scope
+from nanobot.security.workspace_access import (
+    bind_workspace_scope,
+    build_workspace_scope,
+    reset_workspace_scope,
+)
 
 _REAL_GETADDRINFO = socket.getaddrinfo
 
@@ -147,6 +151,7 @@ async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):
             return FakeResponse()
 
     monkeypatch.setattr(tool, "_fetch_jina", _fail_jina)
+    monkeypatch.setattr(tool, "_extract_readable_html", lambda html, mode: "Hello world")
     monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient)
 
     with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
@@ -160,6 +165,47 @@ async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):
     ]
 
 
+@pytest.mark.asyncio
+async def test_web_fetch_falls_back_when_readability_dependency_is_missing(monkeypatch):
+    tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))
+
+    class FakeResponse:
+        status_code = 200
+        url = "https://example.com/page"
+        text = "<html><head><title>Test</title></head><body><p>Hello world</p></body></html>"
+        headers = {"content-type": "text/html"}
+
+        def raise_for_status(self):
+            return None
+
+    class FakeClient:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, exc_type, exc, tb):
+            return False
+
+        async def get(self, url, headers=None, follow_redirects=False, **kwargs):
+            return FakeResponse()
+
+    def _missing_readability(*args, **kwargs):
+        raise ModuleNotFoundError("No module named 'lxml_html_clean'")
+
+    monkeypatch.setattr(tool, "_extract_readable_html", _missing_readability)
+    monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient)
+
+    with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
+        result = await tool._fetch_readability("https://example.com/page", "markdown", 5000)
+
+    data = json.loads(result)
+    assert data["extractor"] == "html"
+    assert data["untrusted"] is True
+    assert "Hello world" in data["text"]
+
+
 @pytest.mark.asyncio
 async def test_web_fetch_blocks_private_redirect_before_readability_request(monkeypatch):
     tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))