From 5f916bbd3afb26c7dd3d9818e261acd9b176f6fa Mon Sep 17 00:00:00 2001 From: Xubin Ren <52506698+Re-bin@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:19:33 +0900 Subject: [PATCH] fix(web): keep credential redirects away from Jina --- nanobot/agent/tools/web.py | 46 ++++++++++----- tests/tools/test_web_fetch_jina_privacy.py | 65 ++++++++++++++++++++++ 2 files changed, 98 insertions(+), 13 deletions(-) diff --git a/nanobot/agent/tools/web.py b/nanobot/agent/tools/web.py index f492dd0f7..f1a2382d7 100644 --- a/nanobot/agent/tools/web.py +++ b/nanobot/agent/tools/web.py @@ -218,13 +218,14 @@ async def _stream_with_safe_redirects( client: httpx.AsyncClient, url: str, headers: dict[str, str] | None = None, -) -> tuple[httpx.Response | None, Any | None, str | None]: +) -> tuple[httpx.Response | None, Any | None, str | None, bool]: """Open a streamed response while validating every redirect target first.""" current_url = url + chain_carries_credentials = _url_carries_credentials(url) for _ in range(MAX_REDIRECTS + 1): is_valid, error_msg, _ = _resolve_url_safe(current_url) if not is_valid: - return None, None, f"Redirect blocked: {error_msg}" + return None, None, f"Redirect blocked: {error_msg}", chain_carries_credentials stream = client.stream( "GET", @@ -237,26 +238,39 @@ async def _stream_with_safe_redirects( except httpx.RequestError as exc: unsafe_error = _unsafe_url_request_error(exc) if unsafe_error is not None: - return None, None, f"Redirect blocked: {unsafe_error}" + return ( + None, + None, + f"Redirect blocked: {unsafe_error}", + chain_carries_credentials, + ) raise is_redirect = 300 <= response.status_code < 400 if not is_redirect: - return response, stream, None + return response, stream, None, chain_carries_credentials location = response.headers.get("location") if not location: - return response, stream, None + return response, stream, None, chain_carries_credentials next_url = urljoin(str(response.url), location) + chain_carries_credentials = ( + chain_carries_credentials or _url_carries_credentials(next_url) + ) is_valid, error_msg = _validate_url_safe(next_url) if not is_valid: await stream.__aexit__(None, None, None) - return None, None, f"Redirect blocked: {error_msg}" + return None, None, f"Redirect blocked: {error_msg}", chain_carries_credentials await stream.__aexit__(None, None, None) current_url = next_url - return None, None, f"Too many redirects: exceeded limit of {MAX_REDIRECTS}" + return ( + None, + None, + f"Too many redirects: exceeded limit of {MAX_REDIRECTS}", + chain_carries_credentials, + ) def _format_results(query: str, items: list[dict[str, Any]], n: int) -> str: @@ -1070,20 +1084,26 @@ class WebFetchTool(Tool): if not is_valid: return json.dumps({"error": f"URL validation failed: {error_msg}", "url": url}, ensure_ascii=False) - # Detect and fetch images directly to avoid Jina's textual image captioning + # Detect and fetch images directly to avoid Jina's textual image captioning. + # This local preflight also proves that no credential-bearing URL occurs + # in the redirect chain before the original URL may be sent to Jina. + jina_remote_safe = False try: async with httpx.AsyncClient( **_fetch_client_kwargs(self.proxy, 15.0), ) as client: - r, stream, redirect_error = await _stream_with_safe_redirects( - client, - url, - headers={"User-Agent": self.user_agent}, + r, stream, redirect_error, chain_carries_credentials = ( + await _stream_with_safe_redirects( + client, + url, + headers={"User-Agent": self.user_agent}, + ) ) if redirect_error: return json.dumps({"error": redirect_error, "url": url}, ensure_ascii=False) if r is None: return json.dumps({"error": "Fetch failed", "url": url}, ensure_ascii=False) + jina_remote_safe = not chain_carries_credentials try: ctype = r.headers.get("content-type", "") @@ -1101,7 +1121,7 @@ class WebFetchTool(Tool): logger.debug("Pre-fetch image detection failed for {}: {}", url, e) result = None - if self.config.use_jina_reader: + if self.config.use_jina_reader and jina_remote_safe: result = await self._fetch_jina(url, max_chars) if result is None: result = await self._fetch_readability(url, extract_mode, max_chars) diff --git a/tests/tools/test_web_fetch_jina_privacy.py b/tests/tools/test_web_fetch_jina_privacy.py index 8f5449842..b49a2350c 100644 --- a/tests/tools/test_web_fetch_jina_privacy.py +++ b/tests/tools/test_web_fetch_jina_privacy.py @@ -179,3 +179,68 @@ async def test_execute_fetches_credential_urls_locally(monkeypatch) -> None: data = json.loads(result) assert data["extractor"] == "readability" assert all("r.jina.ai" not in url for url in requested) + + +async def test_execute_does_not_send_redirected_credential_url_to_jina(monkeypatch) -> None: + """A plain short URL that redirects through a signed URL must stay local.""" + + tool = WebFetchTool() + requested: list[str] = [] + short_url = "https://example.com/short" + signed_url = "https://cdn.example.com/file?token=secret" + + class FakeStreamResponse: + def __init__(self, url: str): + self.url = url + self.status_code = 302 if url == short_url else 200 + self.headers = ( + {"location": signed_url} + if url == short_url + else {"content-type": "text/html"} + ) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + class FakeResponse: + status_code = 200 + url = signed_url + text = "T

ok

" + headers = {"content-type": "text/html"} + is_redirect = False + + def raise_for_status(self): + return None + + class FakeClient: + def __init__(self, *args, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + def stream(self, method, url, headers=None, **kwargs): + requested.append(str(url)) + return FakeStreamResponse(str(url)) + + async def get(self, url, headers=None, **kwargs): + requested.append(str(url)) + return FakeResponse() + + monkeypatch.setattr(tool, "_extract_readable_html", lambda html, mode: "ok") + monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient) + monkeypatch.setattr(web_module, "_pinned_dns_transport", lambda: object()) + + with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public): + result = await tool.execute(url=short_url) + + data = json.loads(result) + assert data["extractor"] == "readability" + assert signed_url in requested + assert all("r.jina.ai" not in url for url in requested)