mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-08-31 08:13:11 +03:00
chengyongru
be3a42ebac
Move blocking filesystem, persistence, subprocess, media, and DNS work off the gateway event loop while preserving existing contracts. Add bounded cancellation and responsiveness regression coverage.
541 lines
20 KiB
Python
541 lines
20 KiB
Python
"""Tests for web_fetch SSRF protection and untrusted content marking."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import socket
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from nanobot.agent.tools import web as web_module
|
|
from nanobot.agent.tools.web import WebFetchTool, _get_with_safe_redirects
|
|
from nanobot.config.schema import WebFetchConfig
|
|
from nanobot.security.network import PinnedDNSAsyncTransport
|
|
from nanobot.security.workspace_access import (
|
|
bind_workspace_scope,
|
|
build_workspace_scope,
|
|
reset_workspace_scope,
|
|
)
|
|
|
|
_REAL_GETADDRINFO = socket.getaddrinfo
|
|
_PROXY_ENV_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy")
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
for name in (*_PROXY_ENV_VARS, "NO_PROXY", "no_proxy"):
|
|
monkeypatch.delenv(name, raising=False)
|
|
|
|
|
|
def _fake_resolve_private(hostname, port, family=0, type_=0, proto=0, flags=0):
|
|
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("169.254.169.254", 0))]
|
|
|
|
|
|
def _fake_resolve_public(hostname, port, family=0, type_=0, proto=0, flags=0):
|
|
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 0))]
|
|
|
|
|
|
def _patch_web_fetch_fake_client(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
|
|
client_kwargs: list[dict] = []
|
|
|
|
class FakeStreamResponse:
|
|
status_code = 200
|
|
headers = {"content-type": "text/html"}
|
|
url = "https://example.com/page"
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
class FakeJinaResponse:
|
|
status_code = 200
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
def json(self):
|
|
return {"data": {"title": "Example", "content": "Hello", "url": "https://example.com/page"}}
|
|
|
|
class FakeClient:
|
|
def __init__(self, *args, **kwargs):
|
|
client_kwargs.append(kwargs)
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def stream(self, method, url, headers=None, **kwargs):
|
|
return FakeStreamResponse()
|
|
|
|
async def get(self, url, headers=None, **kwargs):
|
|
return FakeJinaResponse()
|
|
|
|
monkeypatch.setattr(web_module.httpx, "AsyncClient", FakeClient)
|
|
monkeypatch.setattr(web_module, "_pinned_dns_transport", lambda: object())
|
|
monkeypatch.setattr(
|
|
"nanobot.security.network.httpx.AsyncHTTPTransport",
|
|
lambda **_kwargs: object(),
|
|
)
|
|
return client_kwargs
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_blocks_private_ip():
|
|
tool = WebFetchTool()
|
|
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_private):
|
|
result = await tool.execute(url="http://169.254.169.254/computeMetadata/v1/")
|
|
data = json.loads(result)
|
|
assert "error" in data
|
|
assert "private" in data["error"].lower() or "blocked" in data["error"].lower()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_blocks_localhost():
|
|
tool = WebFetchTool()
|
|
def _resolve_localhost(hostname, port, family=0, type_=0, proto=0, flags=0):
|
|
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("127.0.0.1", 0))]
|
|
with patch("nanobot.security.network.socket.getaddrinfo", _resolve_localhost):
|
|
result = await tool.execute(url="http://localhost/admin")
|
|
data = json.loads(result)
|
|
assert "error" in data
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_blocks_localhost_even_in_full_workspace_scope(tmp_path):
|
|
tool = WebFetchTool()
|
|
scope = build_workspace_scope(tmp_path, "full")
|
|
|
|
def _resolve_localhost(hostname, port, family=0, type_=0, proto=0, flags=0):
|
|
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("127.0.0.1", 0))]
|
|
|
|
token = bind_workspace_scope(scope)
|
|
try:
|
|
with patch("nanobot.security.network.socket.getaddrinfo", _resolve_localhost):
|
|
result = await tool.execute(url="http://localhost/admin")
|
|
finally:
|
|
reset_workspace_scope(token)
|
|
data = json.loads(result)
|
|
assert "error" in data
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_result_contains_untrusted_flag(monkeypatch: pytest.MonkeyPatch):
|
|
"""When fetch succeeds, result JSON must include untrusted=True and the banner."""
|
|
tool = WebFetchTool()
|
|
_patch_web_fetch_fake_client(monkeypatch)
|
|
|
|
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
|
|
result = await tool.execute(url="https://example.com/page")
|
|
|
|
data = json.loads(result)
|
|
assert data.get("untrusted") is True
|
|
assert "[External content" in data.get("text", "")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_safe_redirect_requests_use_independent_pinned_dns_concurrently(monkeypatch):
|
|
public_ips = {
|
|
"a.example": "93.184.216.34",
|
|
"b.example": "93.184.216.35",
|
|
}
|
|
calls: dict[str, int] = {host: 0 for host in public_ips}
|
|
seen: dict[str, str] = {}
|
|
|
|
def _rebinding_resolver(hostname, port, family=0, type_=0, proto=0, flags=0):
|
|
host = str(hostname).rstrip(".").lower()
|
|
calls[host] += 1
|
|
ip = public_ips[host] if calls[host] <= 2 else "169.254.169.254"
|
|
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (ip, 0))]
|
|
|
|
class ResolvingTransport(httpx.AsyncBaseTransport):
|
|
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
await asyncio.sleep(0)
|
|
infos = socket.getaddrinfo(
|
|
request.url.host,
|
|
request.url.port or 443,
|
|
socket.AF_UNSPEC,
|
|
socket.SOCK_STREAM,
|
|
)
|
|
seen[str(request.url)] = infos[0][4][0]
|
|
return httpx.Response(200, request=request)
|
|
|
|
async def _fetch(url: str) -> tuple[httpx.Response | None, str | None]:
|
|
async with httpx.AsyncClient(
|
|
transport=PinnedDNSAsyncTransport(inner=ResolvingTransport())
|
|
) as client:
|
|
return await _get_with_safe_redirects(client, url)
|
|
|
|
monkeypatch.setattr("nanobot.security.network.socket.getaddrinfo", _rebinding_resolver)
|
|
|
|
results = await asyncio.gather(
|
|
_fetch("https://a.example/"),
|
|
_fetch("https://b.example/"),
|
|
)
|
|
|
|
assert all(error is None and response is not None for response, error in results)
|
|
assert seen == {
|
|
"https://a.example/": "93.184.216.34",
|
|
"https://b.example/": "93.184.216.35",
|
|
}
|
|
assert calls == {"a.example": 2, "b.example": 2}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_proxy_remains_supported(monkeypatch):
|
|
tool = WebFetchTool(proxy="http://config-proxy.example:7890")
|
|
client_kwargs = _patch_web_fetch_fake_client(monkeypatch)
|
|
|
|
monkeypatch.setenv("HTTPS_PROXY", "http://env-proxy.example:8080")
|
|
monkeypatch.setenv("NO_PROXY", "example.com")
|
|
|
|
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
|
|
result = await tool.execute(url="https://example.com/page")
|
|
|
|
data = json.loads(result)
|
|
assert data["extractor"] == "jina"
|
|
assert all(kwargs["proxy"] == "http://config-proxy.example:7890" for kwargs in client_kwargs)
|
|
assert all("mounts" not in kwargs for kwargs in client_kwargs)
|
|
assert all("transport" not in kwargs for kwargs in client_kwargs)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_env_proxy_adds_proxy_mounts_and_keeps_pinned_transport(monkeypatch):
|
|
tool = WebFetchTool()
|
|
client_kwargs = _patch_web_fetch_fake_client(monkeypatch)
|
|
|
|
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080")
|
|
monkeypatch.setenv("NO_PROXY", "localhost,127.0.0.1,::1")
|
|
|
|
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
|
|
result = await tool.execute(url="https://example.com/page")
|
|
|
|
data = json.loads(result)
|
|
assert data["extractor"] == "jina"
|
|
fetch_kwargs = [kwargs for kwargs in client_kwargs if kwargs.get("timeout") == 15.0]
|
|
assert fetch_kwargs
|
|
assert all("transport" in kwargs for kwargs in fetch_kwargs)
|
|
assert all("mounts" in kwargs for kwargs in fetch_kwargs)
|
|
|
|
|
|
def test_web_fetch_no_proxy_env_keeps_pinned_direct_route(monkeypatch):
|
|
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080")
|
|
monkeypatch.setenv("NO_PROXY", "example.com")
|
|
monkeypatch.setattr(web_module, "_pinned_dns_transport", lambda: object())
|
|
monkeypatch.setattr(
|
|
"nanobot.security.network.httpx.AsyncHTTPTransport",
|
|
lambda **_kwargs: object(),
|
|
)
|
|
|
|
kwargs = web_module._fetch_client_kwargs(None, 15.0)
|
|
|
|
assert "transport" in kwargs
|
|
assert any(transport is None for transport in kwargs["mounts"].values())
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_does_not_fallback_after_pinned_dns_rebind_rejection(monkeypatch):
|
|
calls = {"evil.example": 0}
|
|
|
|
def _rebinding_resolver(hostname, port, family=0, type_=0, proto=0, flags=0):
|
|
host = str(hostname).rstrip(".").lower()
|
|
calls[host] += 1
|
|
ip = "93.184.216.34" if calls[host] <= 2 else "169.254.169.254"
|
|
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (ip, 0))]
|
|
|
|
tool = WebFetchTool()
|
|
|
|
async def _unexpected_jina(*args, **kwargs):
|
|
raise AssertionError("Jina fallback should not run after an SSRF rejection")
|
|
|
|
async def _unexpected_readability(*args, **kwargs):
|
|
raise AssertionError("Readability fallback should not run after an SSRF rejection")
|
|
|
|
monkeypatch.setattr(tool, "_fetch_jina", _unexpected_jina)
|
|
monkeypatch.setattr(tool, "_fetch_readability", _unexpected_readability)
|
|
|
|
class FailTransport(httpx.AsyncBaseTransport):
|
|
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
raise AssertionError("rebound target must be rejected before transport")
|
|
|
|
monkeypatch.setattr(
|
|
web_module,
|
|
"_pinned_dns_transport",
|
|
lambda: PinnedDNSAsyncTransport(inner=FailTransport()),
|
|
)
|
|
|
|
with patch("nanobot.security.network.socket.getaddrinfo", _rebinding_resolver):
|
|
result = await tool.execute(url="http://evil.example/page")
|
|
|
|
data = json.loads(result)
|
|
assert "error" in data
|
|
assert "blocked" in data["error"].lower()
|
|
assert calls["evil.example"] == 3
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):
|
|
tool = WebFetchTool(
|
|
config=WebFetchConfig(use_jina_reader=False),
|
|
user_agent="nanobot-test-agent",
|
|
)
|
|
seen_headers: list[dict] = []
|
|
|
|
async def _fail_jina(*args, **kwargs):
|
|
raise AssertionError("Jina Reader should be skipped when disabled")
|
|
|
|
class FakeStreamResponse:
|
|
status_code = 200
|
|
headers = {"content-type": "text/html"}
|
|
url = "https://example.com/page"
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
async def aread(self):
|
|
raise AssertionError("non-image prefetch body should not be read")
|
|
|
|
class FakeResponse:
|
|
status_code = 200
|
|
url = "https://example.com/page"
|
|
text = "<html><head><title>Test</title></head><body><p>Hello world</p></body></html>"
|
|
headers = {"content-type": "text/html"}
|
|
is_redirect = False
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
class FakeClient:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def stream(self, method, url, headers=None, **kwargs):
|
|
seen_headers.append(headers or {})
|
|
return FakeStreamResponse()
|
|
|
|
async def get(self, url, headers=None, **kwargs):
|
|
seen_headers.append(headers or {})
|
|
return FakeResponse()
|
|
|
|
monkeypatch.setattr(tool, "_fetch_jina", _fail_jina)
|
|
monkeypatch.setattr(tool, "_extract_readable_html", lambda html, mode: "Hello world")
|
|
monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient)
|
|
monkeypatch.setattr(web_module, "_pinned_dns_transport", lambda: object())
|
|
|
|
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
|
|
result = await tool.execute(url="https://example.com/page")
|
|
|
|
data = json.loads(result)
|
|
assert data["extractor"] == "readability"
|
|
assert [headers["User-Agent"] for headers in seen_headers] == [
|
|
"nanobot-test-agent",
|
|
"nanobot-test-agent",
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_falls_back_when_readability_dependency_is_missing(monkeypatch):
|
|
tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))
|
|
|
|
class FakeResponse:
|
|
status_code = 200
|
|
url = "https://example.com/page"
|
|
text = "<html><head><title>Test</title></head><body><p>Hello world</p></body></html>"
|
|
headers = {"content-type": "text/html"}
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
class FakeClient:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
async def get(self, url, headers=None, follow_redirects=False, **kwargs):
|
|
return FakeResponse()
|
|
|
|
def _missing_readability(*args, **kwargs):
|
|
raise ModuleNotFoundError("No module named 'lxml_html_clean'")
|
|
|
|
monkeypatch.setattr(tool, "_extract_readable_html", _missing_readability)
|
|
monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient)
|
|
monkeypatch.setattr(web_module, "_pinned_dns_transport", lambda: object())
|
|
|
|
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
|
|
result = await tool._fetch_readability("https://example.com/page", "markdown", 5000)
|
|
|
|
data = json.loads(result)
|
|
assert data["extractor"] == "html"
|
|
assert data["untrusted"] is True
|
|
assert "Hello world" in data["text"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_blocks_private_redirect_before_readability_request(monkeypatch):
|
|
tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))
|
|
requested: list[str] = []
|
|
|
|
class FakeStreamResponse:
|
|
status_code = 200
|
|
headers = {"content-type": "text/html"}
|
|
url = "https://attacker.example/start"
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
async def aread(self):
|
|
raise AssertionError("non-image prefetch body should not be read")
|
|
|
|
class FakeRedirectResponse:
|
|
status_code = 302
|
|
headers = {"location": "http://127.0.0.1:8765/metadata"}
|
|
url = "https://attacker.example/start"
|
|
|
|
async def aclose(self):
|
|
return None
|
|
|
|
class FakeClient:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def stream(self, method, url, headers=None, **kwargs):
|
|
return FakeStreamResponse()
|
|
|
|
async def get(self, url, headers=None, **kwargs):
|
|
requested.append(url)
|
|
if url == "http://127.0.0.1:8765/metadata":
|
|
raise AssertionError("private redirect target should not be requested")
|
|
return FakeRedirectResponse()
|
|
|
|
monkeypatch.setattr(web_module.httpx, "AsyncClient", FakeClient)
|
|
monkeypatch.setattr(web_module, "_pinned_dns_transport", lambda: object())
|
|
|
|
def resolve_public_start_only(hostname, port, family=0, type_=0, proto=0, flags=0):
|
|
if hostname == "attacker.example":
|
|
return _fake_resolve_public(hostname, port, family, type_)
|
|
return _REAL_GETADDRINFO(hostname, port, family, type_)
|
|
|
|
with patch("nanobot.security.network.socket.getaddrinfo", resolve_public_start_only):
|
|
result = await tool.execute(url="https://attacker.example/start")
|
|
|
|
data = json.loads(result)
|
|
assert "error" in data
|
|
assert "redirect blocked" in data["error"].lower()
|
|
assert requested == ["https://attacker.example/start"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_blocks_private_redirect_before_returning_image(monkeypatch):
|
|
tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if str(request.url) == "https://example.com/image.png":
|
|
return httpx.Response(
|
|
302,
|
|
headers={"Location": "http://127.0.0.1/secret.png"},
|
|
request=request,
|
|
)
|
|
if str(request.url) == "http://127.0.0.1/secret.png":
|
|
return httpx.Response(
|
|
200,
|
|
headers={"content-type": "image/png"},
|
|
content=b"\x89PNG\r\n\x1a\n",
|
|
request=request,
|
|
)
|
|
return httpx.Response(404, request=request)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
real_async_client = httpx.AsyncClient
|
|
|
|
class TransportAsyncClient(real_async_client):
|
|
def __init__(self, *args, **kwargs):
|
|
kwargs.pop("proxy", None)
|
|
kwargs.pop("transport", None)
|
|
super().__init__(*args, transport=transport, **kwargs)
|
|
|
|
monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", TransportAsyncClient)
|
|
monkeypatch.setattr(web_module, "_pinned_dns_transport", lambda: object())
|
|
|
|
def resolve_public_start_only(hostname, port, family=0, type_=0, proto=0, flags=0):
|
|
if hostname == "example.com":
|
|
return _fake_resolve_public(hostname, port, family, type_)
|
|
return _REAL_GETADDRINFO(hostname, port, family, type_)
|
|
|
|
with patch("nanobot.security.network.socket.getaddrinfo", resolve_public_start_only):
|
|
result = await tool.execute(url="https://example.com/image.png")
|
|
|
|
data = json.loads(result)
|
|
assert "error" in data
|
|
assert "redirect blocked" in data["error"].lower()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_fetch_does_not_request_private_redirect_target(monkeypatch):
|
|
tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))
|
|
requested: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
requested.append(str(request.url))
|
|
if str(request.url) == "https://attacker.example/start":
|
|
return httpx.Response(
|
|
302,
|
|
headers={"Location": "http://127.0.0.1:8765/metadata"},
|
|
request=request,
|
|
)
|
|
if str(request.url) == "http://127.0.0.1:8765/metadata":
|
|
return httpx.Response(200, content=b"internal secret", request=request)
|
|
return httpx.Response(404, request=request)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
real_async_client = httpx.AsyncClient
|
|
|
|
class TransportAsyncClient(real_async_client):
|
|
def __init__(self, *args, **kwargs):
|
|
kwargs["transport"] = transport
|
|
super().__init__(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(web_module.httpx, "AsyncClient", TransportAsyncClient)
|
|
monkeypatch.setattr(web_module, "_pinned_dns_transport", lambda: object())
|
|
|
|
def resolve_public_start_only(hostname, port, family=0, type_=0, proto=0, flags=0):
|
|
if hostname == "attacker.example":
|
|
return _fake_resolve_public(hostname, port, family, type_)
|
|
return _REAL_GETADDRINFO(hostname, port, family, type_)
|
|
|
|
with patch("nanobot.security.network.socket.getaddrinfo", resolve_public_start_only):
|
|
result = await tool.execute(url="https://attacker.example/start")
|
|
|
|
data = json.loads(result)
|
|
assert "error" in data
|
|
assert "redirect blocked" in data["error"].lower()
|
|
assert requested == ["https://attacker.example/start"]
|