fix: pin validated DNS for SSRF-safe fetches

maintainer edit: keep MCP HTTP SSRF checks strict, pin validated DNS for direct web_fetch and HTTP/SSE MCP requests, preserve explicit and environment proxy compatibility, and cover the proxy/redirect/rebinding cases with tests.
2026-07-07 15:40:53 +08:00
parent b68ae4f9bc
commit c5e053f83b
8 changed files with 436 additions and 34 deletions
@@ -25,6 +25,8 @@ from nanobot.bus.events import (
 )
 from nanobot.security.network import (
    PinnedDNSAsyncTransport,
+    env_proxy_applies_to_url,
+    httpx_env_proxy_mounts,
    resolve_url_target,
    validate_url_target,
 )
@@ -179,21 +181,24 @@ async def _probe_http_url(url: str, timeout: float = 3.0) -> bool:
    port = parsed.port
    if not port:
        port = 443 if parsed.scheme == "https" else 80
-    ok, _, resolved_ips = resolve_url_target(url, allow_loopback=True)
+    ok, _, resolved_ips = resolve_url_target(url)
    if not ok:
        return False
-    try:
-        target_host = resolved_ips[0] if resolved_ips else host
-        reader, writer = await asyncio.wait_for(
-            asyncio.open_connection(target_host, port),
-            timeout=timeout,
-        )
-        writer.close()
-        with suppress(OSError, asyncio.TimeoutError):
-            await asyncio.wait_for(writer.wait_closed(), timeout=0.2)
+    if env_proxy_applies_to_url(url):
        return True
-    except (OSError, asyncio.TimeoutError):
-        return False
+    for target_host in resolved_ips or (host,):
+        try:
+            _reader, writer = await asyncio.wait_for(
+                asyncio.open_connection(target_host, port),
+                timeout=timeout,
+            )
+            writer.close()
+            with suppress(OSError, asyncio.TimeoutError):
+                await asyncio.wait_for(writer.wait_closed(), timeout=0.2)
+            return True
+        except (OSError, asyncio.TimeoutError):
+            continue
+    return False


 def _redact_url(url: str) -> str:
@@ -215,9 +220,17 @@ def _redact_url(url: str) -> str:
        return "<redacted-url>"


+def _pinned_transport_kwargs() -> dict[str, object]:
+    kwargs: dict[str, object] = {"transport": PinnedDNSAsyncTransport()}
+    mounts = httpx_env_proxy_mounts()
+    if mounts:
+        kwargs["mounts"] = mounts
+    return kwargs
+
+
 async def _validate_mcp_request_url(request: httpx.Request) -> None:
    """Validate each outgoing MCP HTTP request, including redirect targets."""
-    ok, error = validate_url_target(str(request.url), allow_loopback=True)
+    ok, error = validate_url_target(str(request.url))
    if not ok:
        raise httpx.RequestError(
            f"Blocked unsafe MCP URL {_redact_url(str(request.url))} ({error})",
@@ -884,7 +897,7 @@ async def connect_mcp_servers(
                        follow_redirects=True,
                        timeout=timeout,
                        auth=auth,
-                        transport=PinnedDNSAsyncTransport(allow_loopback=True),
+                        **_pinned_transport_kwargs(),
                    )

                read, write = await server_stack.enter_async_context(
@@ -902,7 +915,7 @@ async def connect_mcp_servers(
                        event_hooks={"request": [_validate_mcp_request_url]},
                        follow_redirects=True,
                        timeout=httpx.Timeout(30.0, connect=10.0),
-                        transport=PinnedDNSAsyncTransport(allow_loopback=True),
+                        **_pinned_transport_kwargs(),
                    )
                )
                read, write, _ = await server_stack.enter_async_context(
@@ -124,6 +124,26 @@ def _pinned_dns_transport() -> httpx.AsyncBaseTransport:
    return PinnedDNSAsyncTransport()


+def _fetch_client_kwargs(proxy: str | None, timeout: float) -> dict[str, Any]:
+    from nanobot.security.network import httpx_env_proxy_mounts
+
+    kwargs: dict[str, Any] = {"timeout": timeout}
+    if proxy:
+        kwargs["proxy"] = proxy
+    else:
+        kwargs["transport"] = _pinned_dns_transport()
+        mounts = httpx_env_proxy_mounts()
+        if mounts:
+            kwargs["mounts"] = mounts
+    return kwargs
+
+
+def _unsafe_url_request_error(exc: BaseException) -> str | None:
+    from nanobot.security.network import UnsafeURLRequestError
+
+    return str(exc) if isinstance(exc, UnsafeURLRequestError) else None
+
+
 async def _get_with_safe_redirects(
    client: httpx.AsyncClient,
    url: str,
@@ -136,7 +156,13 @@ async def _get_with_safe_redirects(
        if not is_valid:
            return None, f"Redirect blocked: {error_msg}"

-        response = await client.get(current_url, headers=headers, follow_redirects=False)
+        try:
+            response = await client.get(current_url, headers=headers, follow_redirects=False)
+        except httpx.RequestError as exc:
+            unsafe_error = _unsafe_url_request_error(exc)
+            if unsafe_error is not None:
+                return None, f"Redirect blocked: {unsafe_error}"
+            raise
        is_redirect = 300 <= response.status_code < 400
        if not is_redirect:
            return response, None
@@ -175,7 +201,13 @@ async def _stream_with_safe_redirects(
            headers=headers,
            follow_redirects=False,
        )
-        response = await stream.__aenter__()
+        try:
+            response = await stream.__aenter__()
+        except httpx.RequestError as exc:
+            unsafe_error = _unsafe_url_request_error(exc)
+            if unsafe_error is not None:
+                return None, None, f"Redirect blocked: {unsafe_error}"
+            raise
        is_redirect = 300 <= response.status_code < 400
        if not is_redirect:
            return response, stream, None
@@ -963,17 +995,11 @@ class WebFetchTool(Tool):
        is_valid, error_msg = _validate_url_safe(url)
        if not is_valid:
            return json.dumps({"error": f"URL validation failed: {error_msg}", "url": url}, ensure_ascii=False)
-        if self.proxy:
-            return json.dumps({
-                "error": "web_fetch proxy is incompatible with DNS-pinned SSRF protection",
-                "url": url,
-            }, ensure_ascii=False)

        # Detect and fetch images directly to avoid Jina's textual image captioning
        try:
            async with httpx.AsyncClient(
-                transport=_pinned_dns_transport(),
-                timeout=15.0,
+                **_fetch_client_kwargs(self.proxy, 15.0),
            ) as client:
                r, stream, redirect_error = await _stream_with_safe_redirects(
                    client,
@@ -995,6 +1021,9 @@ class WebFetchTool(Tool):
                    if stream is not None:
                        await stream.__aexit__(None, None, None)
        except Exception as e:
+            unsafe_error = _unsafe_url_request_error(e)
+            if unsafe_error is not None:
+                return json.dumps({"error": f"URL validation failed: {unsafe_error}", "url": url}, ensure_ascii=False)
            logger.debug("Pre-fetch image detection failed for {}: {}", url, e)

        result = None
@@ -1044,8 +1073,7 @@ class WebFetchTool(Tool):
        """Local fallback using readability-lxml."""
        try:
            async with httpx.AsyncClient(
-                timeout=30.0,
-                transport=_pinned_dns_transport(),
+                **_fetch_client_kwargs(self.proxy, 30.0),
            ) as client:
                r, redirect_error = await _get_with_safe_redirects(
                    client,
@@ -8,6 +8,7 @@ import re
 import socket
 from contextlib import contextmanager, suppress
 from urllib.parse import urlparse
+from urllib.request import getproxies, proxy_bypass

 import httpx

@@ -25,7 +26,6 @@ _BLOCKED_NETWORKS = [
 ]

 _URL_RE = re.compile(r"https?://[^\s\"'`;|<>]+", re.IGNORECASE)
-
 _allowed_networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = []


@@ -113,6 +113,66 @@ def validate_url_target(url: str, *, allow_loopback: bool = False) -> tuple[bool
    return ok, error


+def env_proxy_applies_to_url(url: str) -> bool:
+    """Return True when process proxy settings would proxy this URL."""
+    try:
+        parsed = urlparse(url)
+    except Exception:
+        return False
+    if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+        return False
+
+    proxies = getproxies()
+    proxy_url = proxies.get(parsed.scheme) or proxies.get("all")
+    if not proxy_url:
+        return False
+
+    host = parsed.hostname
+    if parsed.port is not None:
+        host = f"[{host}]:{parsed.port}" if ":" in host else f"{host}:{parsed.port}"
+    return not proxy_bypass(host)
+
+
+def httpx_env_proxy_mounts() -> dict[str, httpx.AsyncBaseTransport | None]:
+    """Build HTTPX proxy mounts while leaving direct routes to the base transport."""
+    proxies = getproxies()
+    mounts: dict[str, httpx.AsyncBaseTransport | None] = {}
+    for scheme in ("http", "https", "all"):
+        proxy_url = proxies.get(scheme)
+        if proxy_url:
+            if "://" not in proxy_url:
+                proxy_url = f"http://{proxy_url}"
+            mounts[f"{scheme}://"] = httpx.AsyncHTTPTransport(proxy=httpx.Proxy(proxy_url))
+
+    if not mounts:
+        return {}
+
+    no_proxy = proxies.get("no", "")
+    if no_proxy == "*":
+        return {}
+    for entry in no_proxy.split(","):
+        pattern = _no_proxy_mount_pattern(entry.strip())
+        if pattern:
+            mounts[pattern] = None
+    return mounts
+
+
+def _no_proxy_mount_pattern(hostname: str) -> str | None:
+    if not hostname:
+        return None
+    if "://" in hostname:
+        return hostname
+
+    unbracketed = hostname.strip("[]")
+    with suppress(ValueError):
+        addr = ipaddress.ip_address(unbracketed)
+        return f"all://[{addr}]" if addr.version == 6 else f"all://{addr}"
+
+    if hostname.lower() == "localhost":
+        return "all://localhost"
+    return f"all://*{hostname}"
+
+
@contextmanager
 def pin_resolved_url_dns(url: str, resolved_ips: tuple[str, ...]):
    """Pin DNS lookups for the URL hostname to previously validated IPs.
@@ -152,6 +212,10 @@ def pin_resolved_url_dns(url: str, resolved_ips: tuple[str, ...]):
        socket.getaddrinfo = original_getaddrinfo


+class UnsafeURLRequestError(httpx.RequestError):
+    """Raised when an outgoing request is rejected by URL safety validation."""
+
+
 class PinnedDNSAsyncTransport(httpx.AsyncBaseTransport):
    """HTTPX transport that pins each request to the IPs validated for its URL."""

@@ -170,7 +234,7 @@ class PinnedDNSAsyncTransport(httpx.AsyncBaseTransport):
        url = str(request.url)
        ok, error, resolved_ips = resolve_url_target(url, allow_loopback=self._allow_loopback)
        if not ok:
-            raise httpx.RequestError(error, request=request)
+            raise UnsafeURLRequestError(error, request=request)
        async with self._resolver_lock:
            with pin_resolved_url_dns(url, resolved_ips):
                return await self._inner.handle_async_request(request)