fix: pin validated DNS for SSRF-safe fetches
maintainer edit: keep MCP HTTP SSRF checks strict, pin validated DNS for direct web_fetch and HTTP/SSE MCP requests, preserve explicit and environment proxy compatibility, and cover the proxy/redirect/rebinding cases with tests.
This commit is contained in:
@@ -1600,18 +1600,21 @@ nanobot uses a shared SSRF guard for built-in web fetches and HTTP/SSE MCP conne
|
|||||||
|
|
||||||
Keep whitelist entries as narrow as possible, such as a single host CIDR (`192.168.1.50/32`). The whitelist is global for the shared SSRF guard; it is not limited to one tool or one MCP server.
|
Keep whitelist entries as narrow as possible, such as a single host CIDR (`192.168.1.50/32`). The whitelist is global for the shared SSRF guard; it is not limited to one tool or one MCP server.
|
||||||
|
|
||||||
|
HTTP/SSE MCP connections use the same process-wide proxy environment behavior as `web_fetch`: proxied targets use the configured proxy, and URLs excluded by `NO_PROXY` remain DNS-pinned direct connections.
|
||||||
|
|
||||||
> [!TIP]
|
> [!TIP]
|
||||||
> Use `proxy` in `tools.web` to route all web requests (search + fetch) through a proxy:
|
> Use `proxy` in `tools.web` to route web requests through a proxy:
|
||||||
> ```json
|
> ```json
|
||||||
> { "tools": { "web": { "proxy": "http://127.0.0.1:7890" } } }
|
> { "tools": { "web": { "proxy": "http://127.0.0.1:7890" } } }
|
||||||
> ```
|
> ```
|
||||||
|
> `web_fetch` applies DNS pinning for direct connections. When an explicit `tools.web.proxy` or a process-wide proxy environment variable applies to the target URL, nanobot still validates the requested URL locally, but DNS resolution for the outbound fetch happens at the proxy; configure only trusted proxies. URLs excluded by `NO_PROXY` keep the DNS-pinned direct path unless `tools.web.proxy` is configured.
|
||||||
|
|
||||||
### `tools.web`
|
### `tools.web`
|
||||||
|
|
||||||
| Option | Type | Default | Description |
|
| Option | Type | Default | Description |
|
||||||
|--------|------|---------|-------------|
|
|--------|------|---------|-------------|
|
||||||
| `enable` | boolean | `true` | Enable or disable all built-in web tools (`web_search` + `web_fetch`) |
|
| `enable` | boolean | `true` | Enable or disable all built-in web tools (`web_search` + `web_fetch`) |
|
||||||
| `proxy` | string or null | `null` | Proxy for all web requests, for example `http://127.0.0.1:7890` |
|
| `proxy` | string or null | `null` | Proxy for web requests, for example `http://127.0.0.1:7890`. `web_fetch` DNS pinning applies only to direct connections; proxied fetches rely on the configured proxy as the trusted network exit. |
|
||||||
| `userAgent` | string or null | `null` | User-Agent header for all web requests. If null, a browser one will be used |
|
| `userAgent` | string or null | `null` | User-Agent header for all web requests. If null, a browser one will be used |
|
||||||
|
|
||||||
### Web Search
|
### Web Search
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ from nanobot.bus.events import (
|
|||||||
)
|
)
|
||||||
from nanobot.security.network import (
|
from nanobot.security.network import (
|
||||||
PinnedDNSAsyncTransport,
|
PinnedDNSAsyncTransport,
|
||||||
|
env_proxy_applies_to_url,
|
||||||
|
httpx_env_proxy_mounts,
|
||||||
resolve_url_target,
|
resolve_url_target,
|
||||||
validate_url_target,
|
validate_url_target,
|
||||||
)
|
)
|
||||||
@@ -179,12 +181,14 @@ async def _probe_http_url(url: str, timeout: float = 3.0) -> bool:
|
|||||||
port = parsed.port
|
port = parsed.port
|
||||||
if not port:
|
if not port:
|
||||||
port = 443 if parsed.scheme == "https" else 80
|
port = 443 if parsed.scheme == "https" else 80
|
||||||
ok, _, resolved_ips = resolve_url_target(url, allow_loopback=True)
|
ok, _, resolved_ips = resolve_url_target(url)
|
||||||
if not ok:
|
if not ok:
|
||||||
return False
|
return False
|
||||||
|
if env_proxy_applies_to_url(url):
|
||||||
|
return True
|
||||||
|
for target_host in resolved_ips or (host,):
|
||||||
try:
|
try:
|
||||||
target_host = resolved_ips[0] if resolved_ips else host
|
_reader, writer = await asyncio.wait_for(
|
||||||
reader, writer = await asyncio.wait_for(
|
|
||||||
asyncio.open_connection(target_host, port),
|
asyncio.open_connection(target_host, port),
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
@@ -193,6 +197,7 @@ async def _probe_http_url(url: str, timeout: float = 3.0) -> bool:
|
|||||||
await asyncio.wait_for(writer.wait_closed(), timeout=0.2)
|
await asyncio.wait_for(writer.wait_closed(), timeout=0.2)
|
||||||
return True
|
return True
|
||||||
except (OSError, asyncio.TimeoutError):
|
except (OSError, asyncio.TimeoutError):
|
||||||
|
continue
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
@@ -215,9 +220,17 @@ def _redact_url(url: str) -> str:
|
|||||||
return "<redacted-url>"
|
return "<redacted-url>"
|
||||||
|
|
||||||
|
|
||||||
|
def _pinned_transport_kwargs() -> dict[str, object]:
|
||||||
|
kwargs: dict[str, object] = {"transport": PinnedDNSAsyncTransport()}
|
||||||
|
mounts = httpx_env_proxy_mounts()
|
||||||
|
if mounts:
|
||||||
|
kwargs["mounts"] = mounts
|
||||||
|
return kwargs
|
||||||
|
|
||||||
|
|
||||||
async def _validate_mcp_request_url(request: httpx.Request) -> None:
|
async def _validate_mcp_request_url(request: httpx.Request) -> None:
|
||||||
"""Validate each outgoing MCP HTTP request, including redirect targets."""
|
"""Validate each outgoing MCP HTTP request, including redirect targets."""
|
||||||
ok, error = validate_url_target(str(request.url), allow_loopback=True)
|
ok, error = validate_url_target(str(request.url))
|
||||||
if not ok:
|
if not ok:
|
||||||
raise httpx.RequestError(
|
raise httpx.RequestError(
|
||||||
f"Blocked unsafe MCP URL {_redact_url(str(request.url))} ({error})",
|
f"Blocked unsafe MCP URL {_redact_url(str(request.url))} ({error})",
|
||||||
@@ -884,7 +897,7 @@ async def connect_mcp_servers(
|
|||||||
follow_redirects=True,
|
follow_redirects=True,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
auth=auth,
|
auth=auth,
|
||||||
transport=PinnedDNSAsyncTransport(allow_loopback=True),
|
**_pinned_transport_kwargs(),
|
||||||
)
|
)
|
||||||
|
|
||||||
read, write = await server_stack.enter_async_context(
|
read, write = await server_stack.enter_async_context(
|
||||||
@@ -902,7 +915,7 @@ async def connect_mcp_servers(
|
|||||||
event_hooks={"request": [_validate_mcp_request_url]},
|
event_hooks={"request": [_validate_mcp_request_url]},
|
||||||
follow_redirects=True,
|
follow_redirects=True,
|
||||||
timeout=httpx.Timeout(30.0, connect=10.0),
|
timeout=httpx.Timeout(30.0, connect=10.0),
|
||||||
transport=PinnedDNSAsyncTransport(allow_loopback=True),
|
**_pinned_transport_kwargs(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
read, write, _ = await server_stack.enter_async_context(
|
read, write, _ = await server_stack.enter_async_context(
|
||||||
|
|||||||
@@ -124,6 +124,26 @@ def _pinned_dns_transport() -> httpx.AsyncBaseTransport:
|
|||||||
return PinnedDNSAsyncTransport()
|
return PinnedDNSAsyncTransport()
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_client_kwargs(proxy: str | None, timeout: float) -> dict[str, Any]:
|
||||||
|
from nanobot.security.network import httpx_env_proxy_mounts
|
||||||
|
|
||||||
|
kwargs: dict[str, Any] = {"timeout": timeout}
|
||||||
|
if proxy:
|
||||||
|
kwargs["proxy"] = proxy
|
||||||
|
else:
|
||||||
|
kwargs["transport"] = _pinned_dns_transport()
|
||||||
|
mounts = httpx_env_proxy_mounts()
|
||||||
|
if mounts:
|
||||||
|
kwargs["mounts"] = mounts
|
||||||
|
return kwargs
|
||||||
|
|
||||||
|
|
||||||
|
def _unsafe_url_request_error(exc: BaseException) -> str | None:
|
||||||
|
from nanobot.security.network import UnsafeURLRequestError
|
||||||
|
|
||||||
|
return str(exc) if isinstance(exc, UnsafeURLRequestError) else None
|
||||||
|
|
||||||
|
|
||||||
async def _get_with_safe_redirects(
|
async def _get_with_safe_redirects(
|
||||||
client: httpx.AsyncClient,
|
client: httpx.AsyncClient,
|
||||||
url: str,
|
url: str,
|
||||||
@@ -136,7 +156,13 @@ async def _get_with_safe_redirects(
|
|||||||
if not is_valid:
|
if not is_valid:
|
||||||
return None, f"Redirect blocked: {error_msg}"
|
return None, f"Redirect blocked: {error_msg}"
|
||||||
|
|
||||||
|
try:
|
||||||
response = await client.get(current_url, headers=headers, follow_redirects=False)
|
response = await client.get(current_url, headers=headers, follow_redirects=False)
|
||||||
|
except httpx.RequestError as exc:
|
||||||
|
unsafe_error = _unsafe_url_request_error(exc)
|
||||||
|
if unsafe_error is not None:
|
||||||
|
return None, f"Redirect blocked: {unsafe_error}"
|
||||||
|
raise
|
||||||
is_redirect = 300 <= response.status_code < 400
|
is_redirect = 300 <= response.status_code < 400
|
||||||
if not is_redirect:
|
if not is_redirect:
|
||||||
return response, None
|
return response, None
|
||||||
@@ -175,7 +201,13 @@ async def _stream_with_safe_redirects(
|
|||||||
headers=headers,
|
headers=headers,
|
||||||
follow_redirects=False,
|
follow_redirects=False,
|
||||||
)
|
)
|
||||||
|
try:
|
||||||
response = await stream.__aenter__()
|
response = await stream.__aenter__()
|
||||||
|
except httpx.RequestError as exc:
|
||||||
|
unsafe_error = _unsafe_url_request_error(exc)
|
||||||
|
if unsafe_error is not None:
|
||||||
|
return None, None, f"Redirect blocked: {unsafe_error}"
|
||||||
|
raise
|
||||||
is_redirect = 300 <= response.status_code < 400
|
is_redirect = 300 <= response.status_code < 400
|
||||||
if not is_redirect:
|
if not is_redirect:
|
||||||
return response, stream, None
|
return response, stream, None
|
||||||
@@ -963,17 +995,11 @@ class WebFetchTool(Tool):
|
|||||||
is_valid, error_msg = _validate_url_safe(url)
|
is_valid, error_msg = _validate_url_safe(url)
|
||||||
if not is_valid:
|
if not is_valid:
|
||||||
return json.dumps({"error": f"URL validation failed: {error_msg}", "url": url}, ensure_ascii=False)
|
return json.dumps({"error": f"URL validation failed: {error_msg}", "url": url}, ensure_ascii=False)
|
||||||
if self.proxy:
|
|
||||||
return json.dumps({
|
|
||||||
"error": "web_fetch proxy is incompatible with DNS-pinned SSRF protection",
|
|
||||||
"url": url,
|
|
||||||
}, ensure_ascii=False)
|
|
||||||
|
|
||||||
# Detect and fetch images directly to avoid Jina's textual image captioning
|
# Detect and fetch images directly to avoid Jina's textual image captioning
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(
|
async with httpx.AsyncClient(
|
||||||
transport=_pinned_dns_transport(),
|
**_fetch_client_kwargs(self.proxy, 15.0),
|
||||||
timeout=15.0,
|
|
||||||
) as client:
|
) as client:
|
||||||
r, stream, redirect_error = await _stream_with_safe_redirects(
|
r, stream, redirect_error = await _stream_with_safe_redirects(
|
||||||
client,
|
client,
|
||||||
@@ -995,6 +1021,9 @@ class WebFetchTool(Tool):
|
|||||||
if stream is not None:
|
if stream is not None:
|
||||||
await stream.__aexit__(None, None, None)
|
await stream.__aexit__(None, None, None)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
unsafe_error = _unsafe_url_request_error(e)
|
||||||
|
if unsafe_error is not None:
|
||||||
|
return json.dumps({"error": f"URL validation failed: {unsafe_error}", "url": url}, ensure_ascii=False)
|
||||||
logger.debug("Pre-fetch image detection failed for {}: {}", url, e)
|
logger.debug("Pre-fetch image detection failed for {}: {}", url, e)
|
||||||
|
|
||||||
result = None
|
result = None
|
||||||
@@ -1044,8 +1073,7 @@ class WebFetchTool(Tool):
|
|||||||
"""Local fallback using readability-lxml."""
|
"""Local fallback using readability-lxml."""
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(
|
async with httpx.AsyncClient(
|
||||||
timeout=30.0,
|
**_fetch_client_kwargs(self.proxy, 30.0),
|
||||||
transport=_pinned_dns_transport(),
|
|
||||||
) as client:
|
) as client:
|
||||||
r, redirect_error = await _get_with_safe_redirects(
|
r, redirect_error = await _get_with_safe_redirects(
|
||||||
client,
|
client,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import re
|
|||||||
import socket
|
import socket
|
||||||
from contextlib import contextmanager, suppress
|
from contextlib import contextmanager, suppress
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
from urllib.request import getproxies, proxy_bypass
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
@@ -25,7 +26,6 @@ _BLOCKED_NETWORKS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
_URL_RE = re.compile(r"https?://[^\s\"'`;|<>]+", re.IGNORECASE)
|
_URL_RE = re.compile(r"https?://[^\s\"'`;|<>]+", re.IGNORECASE)
|
||||||
|
|
||||||
_allowed_networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = []
|
_allowed_networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = []
|
||||||
|
|
||||||
|
|
||||||
@@ -113,6 +113,66 @@ def validate_url_target(url: str, *, allow_loopback: bool = False) -> tuple[bool
|
|||||||
return ok, error
|
return ok, error
|
||||||
|
|
||||||
|
|
||||||
|
def env_proxy_applies_to_url(url: str) -> bool:
|
||||||
|
"""Return True when process proxy settings would proxy this URL."""
|
||||||
|
try:
|
||||||
|
parsed = urlparse(url)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||||
|
return False
|
||||||
|
|
||||||
|
proxies = getproxies()
|
||||||
|
proxy_url = proxies.get(parsed.scheme) or proxies.get("all")
|
||||||
|
if not proxy_url:
|
||||||
|
return False
|
||||||
|
|
||||||
|
host = parsed.hostname
|
||||||
|
if parsed.port is not None:
|
||||||
|
host = f"[{host}]:{parsed.port}" if ":" in host else f"{host}:{parsed.port}"
|
||||||
|
return not proxy_bypass(host)
|
||||||
|
|
||||||
|
|
||||||
|
def httpx_env_proxy_mounts() -> dict[str, httpx.AsyncBaseTransport | None]:
|
||||||
|
"""Build HTTPX proxy mounts while leaving direct routes to the base transport."""
|
||||||
|
proxies = getproxies()
|
||||||
|
mounts: dict[str, httpx.AsyncBaseTransport | None] = {}
|
||||||
|
for scheme in ("http", "https", "all"):
|
||||||
|
proxy_url = proxies.get(scheme)
|
||||||
|
if proxy_url:
|
||||||
|
if "://" not in proxy_url:
|
||||||
|
proxy_url = f"http://{proxy_url}"
|
||||||
|
mounts[f"{scheme}://"] = httpx.AsyncHTTPTransport(proxy=httpx.Proxy(proxy_url))
|
||||||
|
|
||||||
|
if not mounts:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
no_proxy = proxies.get("no", "")
|
||||||
|
if no_proxy == "*":
|
||||||
|
return {}
|
||||||
|
for entry in no_proxy.split(","):
|
||||||
|
pattern = _no_proxy_mount_pattern(entry.strip())
|
||||||
|
if pattern:
|
||||||
|
mounts[pattern] = None
|
||||||
|
return mounts
|
||||||
|
|
||||||
|
|
||||||
|
def _no_proxy_mount_pattern(hostname: str) -> str | None:
|
||||||
|
if not hostname:
|
||||||
|
return None
|
||||||
|
if "://" in hostname:
|
||||||
|
return hostname
|
||||||
|
|
||||||
|
unbracketed = hostname.strip("[]")
|
||||||
|
with suppress(ValueError):
|
||||||
|
addr = ipaddress.ip_address(unbracketed)
|
||||||
|
return f"all://[{addr}]" if addr.version == 6 else f"all://{addr}"
|
||||||
|
|
||||||
|
if hostname.lower() == "localhost":
|
||||||
|
return "all://localhost"
|
||||||
|
return f"all://*{hostname}"
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def pin_resolved_url_dns(url: str, resolved_ips: tuple[str, ...]):
|
def pin_resolved_url_dns(url: str, resolved_ips: tuple[str, ...]):
|
||||||
"""Pin DNS lookups for the URL hostname to previously validated IPs.
|
"""Pin DNS lookups for the URL hostname to previously validated IPs.
|
||||||
@@ -152,6 +212,10 @@ def pin_resolved_url_dns(url: str, resolved_ips: tuple[str, ...]):
|
|||||||
socket.getaddrinfo = original_getaddrinfo
|
socket.getaddrinfo = original_getaddrinfo
|
||||||
|
|
||||||
|
|
||||||
|
class UnsafeURLRequestError(httpx.RequestError):
|
||||||
|
"""Raised when an outgoing request is rejected by URL safety validation."""
|
||||||
|
|
||||||
|
|
||||||
class PinnedDNSAsyncTransport(httpx.AsyncBaseTransport):
|
class PinnedDNSAsyncTransport(httpx.AsyncBaseTransport):
|
||||||
"""HTTPX transport that pins each request to the IPs validated for its URL."""
|
"""HTTPX transport that pins each request to the IPs validated for its URL."""
|
||||||
|
|
||||||
@@ -170,7 +234,7 @@ class PinnedDNSAsyncTransport(httpx.AsyncBaseTransport):
|
|||||||
url = str(request.url)
|
url = str(request.url)
|
||||||
ok, error, resolved_ips = resolve_url_target(url, allow_loopback=self._allow_loopback)
|
ok, error, resolved_ips = resolve_url_target(url, allow_loopback=self._allow_loopback)
|
||||||
if not ok:
|
if not ok:
|
||||||
raise httpx.RequestError(error, request=request)
|
raise UnsafeURLRequestError(error, request=request)
|
||||||
async with self._resolver_lock:
|
async with self._resolver_lock:
|
||||||
with pin_resolved_url_dns(url, resolved_ips):
|
with pin_resolved_url_dns(url, resolved_ips):
|
||||||
return await self._inner.handle_async_request(request)
|
return await self._inner.handle_async_request(request)
|
||||||
|
|||||||
@@ -11,11 +11,21 @@ import pytest
|
|||||||
from nanobot.security.network import (
|
from nanobot.security.network import (
|
||||||
configure_ssrf_whitelist,
|
configure_ssrf_whitelist,
|
||||||
contains_internal_url,
|
contains_internal_url,
|
||||||
|
env_proxy_applies_to_url,
|
||||||
|
httpx_env_proxy_mounts,
|
||||||
pin_resolved_url_dns,
|
pin_resolved_url_dns,
|
||||||
resolve_url_target,
|
resolve_url_target,
|
||||||
validate_url_target,
|
validate_url_target,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_PROXY_ENV_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
for name in (*_PROXY_ENV_VARS, "NO_PROXY", "no_proxy"):
|
||||||
|
monkeypatch.delenv(name, raising=False)
|
||||||
|
|
||||||
|
|
||||||
def _fake_resolve(host: str, results: list[str]):
|
def _fake_resolve(host: str, results: list[str]):
|
||||||
"""Return a getaddrinfo mock that maps the given host to fake IP results."""
|
"""Return a getaddrinfo mock that maps the given host to fake IP results."""
|
||||||
@@ -184,6 +194,18 @@ def test_allows_normal_https():
|
|||||||
assert ok
|
assert ok
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_proxy_helpers_respect_no_proxy(monkeypatch):
|
||||||
|
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080")
|
||||||
|
monkeypatch.setenv("NO_PROXY", "localhost,127.0.0.1,::1")
|
||||||
|
|
||||||
|
assert env_proxy_applies_to_url("https://example.com/page")
|
||||||
|
assert not env_proxy_applies_to_url("http://localhost:8765/mcp")
|
||||||
|
|
||||||
|
mounts = httpx_env_proxy_mounts()
|
||||||
|
assert any(transport is None for transport in mounts.values())
|
||||||
|
assert any(transport is not None for transport in mounts.values())
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# contains_internal_url — shell command scanning
|
# contains_internal_url — shell command scanning
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -9,6 +9,16 @@ import pytest
|
|||||||
|
|
||||||
from nanobot.agent.tools.mcp import _probe_http_url, connect_mcp_servers
|
from nanobot.agent.tools.mcp import _probe_http_url, connect_mcp_servers
|
||||||
from nanobot.agent.tools.registry import ToolRegistry
|
from nanobot.agent.tools.registry import ToolRegistry
|
||||||
|
from nanobot.security.network import configure_ssrf_whitelist
|
||||||
|
|
||||||
|
_PROXY_ENV_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
for name in (*_PROXY_ENV_VARS, "NO_PROXY", "no_proxy"):
|
||||||
|
monkeypatch.delenv(name, raising=False)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# _probe_http_url unit tests
|
# _probe_http_url unit tests
|
||||||
@@ -23,9 +33,11 @@ async def test_probe_returns_true_for_open_port(tmp_path):
|
|||||||
|
|
||||||
server = await asyncio.start_server(_close_connection, "127.0.0.1", 0)
|
server = await asyncio.start_server(_close_connection, "127.0.0.1", 0)
|
||||||
port = server.sockets[0].getsockname()[1]
|
port = server.sockets[0].getsockname()[1]
|
||||||
|
configure_ssrf_whitelist(["127.0.0.1/32"])
|
||||||
try:
|
try:
|
||||||
assert await _probe_http_url(f"http://127.0.0.1:{port}/mcp") is True
|
assert await _probe_http_url(f"http://127.0.0.1:{port}/mcp") is True
|
||||||
finally:
|
finally:
|
||||||
|
configure_ssrf_whitelist([])
|
||||||
server.close()
|
server.close()
|
||||||
await server.wait_closed()
|
await server.wait_closed()
|
||||||
|
|
||||||
@@ -51,6 +63,55 @@ async def test_probe_rejects_public_name_resolving_to_loopback():
|
|||||||
assert await _probe_http_url("http://example.com:8765/mcp") is False
|
assert await _probe_http_url("http://example.com:8765/mcp") is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_probe_skips_direct_tcp_when_global_proxy_env_is_set(monkeypatch):
|
||||||
|
def _resolver(hostname, port, family=0, type_=0):
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 0))]
|
||||||
|
|
||||||
|
async def _open_connection(*args, **kwargs):
|
||||||
|
raise AssertionError("global proxy env should skip direct TCP probe")
|
||||||
|
|
||||||
|
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080")
|
||||||
|
monkeypatch.setenv("NO_PROXY", "localhost,127.0.0.1,::1")
|
||||||
|
monkeypatch.setattr("nanobot.agent.tools.mcp.asyncio.open_connection", _open_connection)
|
||||||
|
|
||||||
|
with patch("nanobot.security.network.socket.getaddrinfo", _resolver):
|
||||||
|
assert await _probe_http_url("https://mcp.example.com/mcp") is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_probe_tries_next_validated_ip_when_first_is_unreachable(monkeypatch):
|
||||||
|
attempts: list[tuple[str, int]] = []
|
||||||
|
|
||||||
|
class FakeWriter:
|
||||||
|
def close(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def wait_closed(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _resolver(hostname, port, family=0, type_=0):
|
||||||
|
return [
|
||||||
|
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 0)),
|
||||||
|
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.35", 0)),
|
||||||
|
]
|
||||||
|
|
||||||
|
async def _open_connection(host: str, port: int):
|
||||||
|
attempts.append((host, port))
|
||||||
|
if host == "93.184.216.34":
|
||||||
|
raise OSError("first address unreachable")
|
||||||
|
return object(), FakeWriter()
|
||||||
|
|
||||||
|
monkeypatch.setattr("nanobot.security.network.socket.getaddrinfo", _resolver)
|
||||||
|
monkeypatch.setattr("nanobot.agent.tools.mcp.asyncio.open_connection", _open_connection)
|
||||||
|
|
||||||
|
assert await _probe_http_url("http://mcp.example:8765/mcp") is True
|
||||||
|
assert attempts == [
|
||||||
|
("93.184.216.34", 8765),
|
||||||
|
("93.184.216.35", 8765),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# connect_mcp_servers skips unreachable HTTP servers
|
# connect_mcp_servers skips unreachable HTTP servers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ from nanobot.agent.tools.mcp import (
|
|||||||
from nanobot.agent.tools.registry import ToolRegistry, is_tool_error_result
|
from nanobot.agent.tools.registry import ToolRegistry, is_tool_error_result
|
||||||
from nanobot.config.schema import MCPServerConfig
|
from nanobot.config.schema import MCPServerConfig
|
||||||
|
|
||||||
|
_PROXY_ENV_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy")
|
||||||
|
|
||||||
|
|
||||||
class _FakeTextContent:
|
class _FakeTextContent:
|
||||||
def __init__(self, text: str) -> None:
|
def __init__(self, text: str) -> None:
|
||||||
@@ -50,6 +52,12 @@ def fake_mcp_runtime() -> dict[str, object | None]:
|
|||||||
return {"session": None}
|
return {"session": None}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
for name in (*_PROXY_ENV_VARS, "NO_PROXY", "no_proxy"):
|
||||||
|
monkeypatch.delenv(name, raising=False)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def _fake_mcp_module(
|
def _fake_mcp_module(
|
||||||
monkeypatch: pytest.MonkeyPatch, fake_mcp_runtime: dict[str, object | None]
|
monkeypatch: pytest.MonkeyPatch, fake_mcp_runtime: dict[str, object | None]
|
||||||
@@ -712,6 +720,8 @@ async def test_connect_mcp_servers_logs_stdio_pollution_hint(
|
|||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"config",
|
"config",
|
||||||
[
|
[
|
||||||
|
MCPServerConfig(url="http://127.0.0.1:9/sse"),
|
||||||
|
MCPServerConfig(type="streamableHttp", url="http://127.0.0.1:9/mcp"),
|
||||||
MCPServerConfig(url="http://169.254.169.254/sse"),
|
MCPServerConfig(url="http://169.254.169.254/sse"),
|
||||||
MCPServerConfig(type="streamableHttp", url="http://169.254.169.254/mcp"),
|
MCPServerConfig(type="streamableHttp", url="http://169.254.169.254/mcp"),
|
||||||
],
|
],
|
||||||
@@ -742,6 +752,93 @@ async def test_connect_mcp_servers_rejects_unsafe_http_urls_before_probe(
|
|||||||
assert any("blocked unsafe URL" in warning for warning in warnings)
|
assert any("blocked unsafe URL" in warning for warning in warnings)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_validate_mcp_request_url_rejects_loopback_without_whitelist() -> None:
|
||||||
|
from nanobot.security.network import configure_ssrf_whitelist
|
||||||
|
|
||||||
|
configure_ssrf_whitelist([])
|
||||||
|
request = httpx.Request("GET", "http://127.0.0.1/private")
|
||||||
|
|
||||||
|
with pytest.raises(httpx.RequestError, match="Blocked unsafe MCP URL"):
|
||||||
|
await mcp_mod._validate_mcp_request_url(request)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"config",
|
||||||
|
[
|
||||||
|
MCPServerConfig(type="sse", url="https://mcp.example.com/sse"),
|
||||||
|
MCPServerConfig(type="streamableHttp", url="https://mcp.example.com/mcp"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
async def test_connect_mcp_servers_env_proxy_adds_proxy_mounts_and_keeps_pinned_transport(
|
||||||
|
config: MCPServerConfig,
|
||||||
|
fake_mcp_runtime: dict[str, object | None],
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
fake_mcp_runtime["session"] = _make_fake_session(["demo"])
|
||||||
|
client_kwargs: list[dict[str, object]] = []
|
||||||
|
|
||||||
|
async def _reachable(_url: str) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _validate(_url: str) -> tuple[bool, str]:
|
||||||
|
return True, ""
|
||||||
|
|
||||||
|
class FakeAsyncClient:
|
||||||
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
||||||
|
client_kwargs.append(kwargs)
|
||||||
|
|
||||||
|
async def __aenter__(self) -> object:
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type: object, exc: object, tb: object) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def _capturing_sse_client(_url: str, httpx_client_factory=None):
|
||||||
|
assert httpx_client_factory is not None
|
||||||
|
async with httpx_client_factory():
|
||||||
|
pass
|
||||||
|
yield object(), object()
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def _capturing_streamable_http_client(_url: str, http_client=None):
|
||||||
|
assert http_client is not None
|
||||||
|
yield object(), object(), object()
|
||||||
|
|
||||||
|
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080")
|
||||||
|
monkeypatch.setenv("NO_PROXY", "localhost,127.0.0.1,::1")
|
||||||
|
monkeypatch.setattr(mcp_mod, "validate_url_target", _validate)
|
||||||
|
monkeypatch.setattr(mcp_mod, "_probe_http_url", _reachable)
|
||||||
|
monkeypatch.setattr(mcp_mod.httpx, "AsyncClient", FakeAsyncClient)
|
||||||
|
monkeypatch.setattr(sys.modules["mcp.client.sse"], "sse_client", _capturing_sse_client)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
sys.modules["mcp.client.streamable_http"],
|
||||||
|
"streamable_http_client",
|
||||||
|
_capturing_streamable_http_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
registry = ToolRegistry()
|
||||||
|
stacks = await connect_mcp_servers({"remote": config}, registry)
|
||||||
|
for stack in stacks.values():
|
||||||
|
await stack.aclose()
|
||||||
|
|
||||||
|
assert client_kwargs
|
||||||
|
assert all("transport" in kwargs for kwargs in client_kwargs)
|
||||||
|
assert all("mounts" in kwargs for kwargs in client_kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mcp_http_clients_no_proxy_env_keeps_pinned_direct_route(monkeypatch):
|
||||||
|
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080")
|
||||||
|
monkeypatch.setenv("NO_PROXY", "mcp.example.com")
|
||||||
|
|
||||||
|
kwargs = mcp_mod._pinned_transport_kwargs()
|
||||||
|
|
||||||
|
assert "transport" in kwargs
|
||||||
|
assert any(transport is None for transport in kwargs["mounts"].values())
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("config", "expected_transport"),
|
("config", "expected_transport"),
|
||||||
|
|||||||
@@ -21,6 +21,13 @@ from nanobot.security.workspace_access import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
_REAL_GETADDRINFO = socket.getaddrinfo
|
_REAL_GETADDRINFO = socket.getaddrinfo
|
||||||
|
_PROXY_ENV_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
for name in (*_PROXY_ENV_VARS, "NO_PROXY", "no_proxy"):
|
||||||
|
monkeypatch.delenv(name, raising=False)
|
||||||
|
|
||||||
|
|
||||||
def _fake_resolve_private(hostname, port, family=0, type_=0):
|
def _fake_resolve_private(hostname, port, family=0, type_=0):
|
||||||
@@ -31,6 +38,49 @@ def _fake_resolve_public(hostname, port, family=0, type_=0):
|
|||||||
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 0))]
|
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 0))]
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_web_fetch_fake_client(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
|
||||||
|
client_kwargs: list[dict] = []
|
||||||
|
|
||||||
|
class FakeStreamResponse:
|
||||||
|
status_code = 200
|
||||||
|
headers = {"content-type": "text/html"}
|
||||||
|
url = "https://example.com/page"
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type, exc, tb):
|
||||||
|
return False
|
||||||
|
|
||||||
|
class FakeJinaResponse:
|
||||||
|
status_code = 200
|
||||||
|
|
||||||
|
def raise_for_status(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return {"data": {"title": "Example", "content": "Hello", "url": "https://example.com/page"}}
|
||||||
|
|
||||||
|
class FakeClient:
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
client_kwargs.append(kwargs)
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type, exc, tb):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def stream(self, method, url, headers=None, **kwargs):
|
||||||
|
return FakeStreamResponse()
|
||||||
|
|
||||||
|
async def get(self, url, headers=None, **kwargs):
|
||||||
|
return FakeJinaResponse()
|
||||||
|
|
||||||
|
monkeypatch.setattr(web_module.httpx, "AsyncClient", FakeClient)
|
||||||
|
return client_kwargs
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_web_fetch_blocks_private_ip():
|
async def test_web_fetch_blocks_private_ip():
|
||||||
tool = WebFetchTool()
|
tool = WebFetchTool()
|
||||||
@@ -148,16 +198,80 @@ async def test_safe_redirect_requests_use_independent_pinned_dns_concurrently(mo
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_web_fetch_rejects_proxy_because_upstream_dns_cannot_be_pinned():
|
async def test_web_fetch_proxy_remains_supported(monkeypatch):
|
||||||
tool = WebFetchTool(proxy="http://proxy.example:8080")
|
tool = WebFetchTool(proxy="http://config-proxy.example:7890")
|
||||||
|
client_kwargs = _patch_web_fetch_fake_client(monkeypatch)
|
||||||
|
|
||||||
|
monkeypatch.setenv("HTTPS_PROXY", "http://env-proxy.example:8080")
|
||||||
|
monkeypatch.setenv("NO_PROXY", "example.com")
|
||||||
|
|
||||||
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
|
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
|
||||||
result = await tool.execute(url="https://example.com/page")
|
result = await tool.execute(url="https://example.com/page")
|
||||||
|
|
||||||
|
data = json.loads(result)
|
||||||
|
assert data["extractor"] == "jina"
|
||||||
|
assert all(kwargs["proxy"] == "http://config-proxy.example:7890" for kwargs in client_kwargs)
|
||||||
|
assert all("mounts" not in kwargs for kwargs in client_kwargs)
|
||||||
|
assert all("transport" not in kwargs for kwargs in client_kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_fetch_env_proxy_adds_proxy_mounts_and_keeps_pinned_transport(monkeypatch):
|
||||||
|
tool = WebFetchTool()
|
||||||
|
client_kwargs = _patch_web_fetch_fake_client(monkeypatch)
|
||||||
|
|
||||||
|
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080")
|
||||||
|
monkeypatch.setenv("NO_PROXY", "localhost,127.0.0.1,::1")
|
||||||
|
|
||||||
|
with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):
|
||||||
|
result = await tool.execute(url="https://example.com/page")
|
||||||
|
|
||||||
|
data = json.loads(result)
|
||||||
|
assert data["extractor"] == "jina"
|
||||||
|
fetch_kwargs = [kwargs for kwargs in client_kwargs if kwargs.get("timeout") == 15.0]
|
||||||
|
assert fetch_kwargs
|
||||||
|
assert all("transport" in kwargs for kwargs in fetch_kwargs)
|
||||||
|
assert all("mounts" in kwargs for kwargs in fetch_kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_web_fetch_no_proxy_env_keeps_pinned_direct_route(monkeypatch):
|
||||||
|
monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080")
|
||||||
|
monkeypatch.setenv("NO_PROXY", "example.com")
|
||||||
|
|
||||||
|
kwargs = web_module._fetch_client_kwargs(None, 15.0)
|
||||||
|
|
||||||
|
assert "transport" in kwargs
|
||||||
|
assert any(transport is None for transport in kwargs["mounts"].values())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_fetch_does_not_fallback_after_pinned_dns_rebind_rejection(monkeypatch):
|
||||||
|
calls = {"evil.example": 0}
|
||||||
|
|
||||||
|
def _rebinding_resolver(hostname, port, family=0, type_=0, proto=0, flags=0):
|
||||||
|
host = str(hostname).rstrip(".").lower()
|
||||||
|
calls[host] += 1
|
||||||
|
ip = "93.184.216.34" if calls[host] <= 2 else "169.254.169.254"
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (ip, 0))]
|
||||||
|
|
||||||
|
tool = WebFetchTool()
|
||||||
|
|
||||||
|
async def _unexpected_jina(*args, **kwargs):
|
||||||
|
raise AssertionError("Jina fallback should not run after an SSRF rejection")
|
||||||
|
|
||||||
|
async def _unexpected_readability(*args, **kwargs):
|
||||||
|
raise AssertionError("Readability fallback should not run after an SSRF rejection")
|
||||||
|
|
||||||
|
monkeypatch.setattr(tool, "_fetch_jina", _unexpected_jina)
|
||||||
|
monkeypatch.setattr(tool, "_fetch_readability", _unexpected_readability)
|
||||||
|
|
||||||
|
with patch("nanobot.security.network.socket.getaddrinfo", _rebinding_resolver):
|
||||||
|
result = await tool.execute(url="http://evil.example/page")
|
||||||
|
|
||||||
data = json.loads(result)
|
data = json.loads(result)
|
||||||
assert "error" in data
|
assert "error" in data
|
||||||
assert "proxy" in data["error"].lower()
|
assert "blocked" in data["error"].lower()
|
||||||
assert "dns-pinned" in data["error"].lower()
|
assert calls["evil.example"] == 3
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
Reference in New Issue
Block a user