feat(web): multi-provider web search + Jina Reader fetch

nanobot/agent/tools/web.py

@@ -1,10 +1,13 @@
 """Web tools: web_search and web_fetch."""
 
+from __future__ import annotations
+
+import asyncio
 import html
 import json
 import os
 import re
-from typing import Any
+from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
 import httpx
@@ -12,6 +15,9 @@ from loguru import logger
 
 from nanobot.agent.tools.base import Tool
 
+if TYPE_CHECKING:
+    from nanobot.config.schema import WebSearchConfig
+
 # Shared constants
 USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36"
 MAX_REDIRECTS = 5  # Limit redirects to prevent DoS attacks
@@ -44,8 +50,22 @@ def _validate_url(url: str) -> tuple[bool, str]:
         return False, str(e)
 
 
+def _format_results(query: str, items: list[dict[str, Any]], n: int) -> str:
+    """Format provider results into shared plaintext output."""
+    if not items:
+        return f"No results for: {query}"
+    lines = [f"Results for: {query}\n"]
+    for i, item in enumerate(items[:n], 1):
+        title = _normalize(_strip_tags(item.get("title", "")))
+        snippet = _normalize(_strip_tags(item.get("content", "")))
+        lines.append(f"{i}. {title}\n   {item.get('url', '')}")
+        if snippet:
+            lines.append(f"   {snippet}")
+    return "\n".join(lines)
+
+
 class WebSearchTool(Tool):
-    """Search the web using Brave Search API."""
+    """Search the web using configured provider."""
 
     name = "web_search"
     description = "Search the web. Returns titles, URLs, and snippets."
@@ -53,61 +73,140 @@ class WebSearchTool(Tool):
         "type": "object",
         "properties": {
             "query": {"type": "string", "description": "Search query"},
-            "count": {"type": "integer", "description": "Results (1-10)", "minimum": 1, "maximum": 10}
+            "count": {"type": "integer", "description": "Results (1-10)", "minimum": 1, "maximum": 10},
         },
-        "required": ["query"]
+        "required": ["query"],
     }
 
-    def __init__(self, api_key: str | None = None, max_results: int = 5, proxy: str | None = None):
-        self._init_api_key = api_key
-        self.max_results = max_results
+    def __init__(self, config: WebSearchConfig | None = None, proxy: str | None = None):
+        from nanobot.config.schema import WebSearchConfig
+
+        self.config = config if config is not None else WebSearchConfig()
         self.proxy = proxy
 
-    @property
-    def api_key(self) -> str:
-        """Resolve API key at call time so env/config changes are picked up."""
-        return self._init_api_key or os.environ.get("BRAVE_API_KEY", "")
-
     async def execute(self, query: str, count: int | None = None, **kwargs: Any) -> str:
-        if not self.api_key:
-            return (
-                "Error: Brave Search API key not configured. Set it in "
-                "~/.nanobot/config.json under tools.web.search.apiKey "
-                "(or export BRAVE_API_KEY), then restart the gateway."
-            )
+        provider = self.config.provider.strip().lower() or "brave"
+        n = min(max(count or self.config.max_results, 1), 10)
 
+        if provider == "duckduckgo":
+            return await self._search_duckduckgo(query, n)
+        elif provider == "tavily":
+            return await self._search_tavily(query, n)
+        elif provider == "searxng":
+            return await self._search_searxng(query, n)
+        elif provider == "jina":
+            return await self._search_jina(query, n)
+        elif provider == "brave":
+            return await self._search_brave(query, n)
+        else:
+            return f"Error: unknown search provider '{provider}'"
+
+    async def _search_brave(self, query: str, n: int) -> str:
+        api_key = self.config.api_key or os.environ.get("BRAVE_API_KEY", "")
+        if not api_key:
+            logger.warning("BRAVE_API_KEY not set, falling back to DuckDuckGo")
+            return await self._search_duckduckgo(query, n)
         try:
-            n = min(max(count or self.max_results, 1), 10)
-            logger.debug("WebSearch: {}", "proxy enabled" if self.proxy else "direct connection")
             async with httpx.AsyncClient(proxy=self.proxy) as client:
                 r = await client.get(
                     "https://api.search.brave.com/res/v1/web/search",
                     params={"q": query, "count": n},
-                    headers={"Accept": "application/json", "X-Subscription-Token": self.api_key},
-                    timeout=10.0
+                    headers={"Accept": "application/json", "X-Subscription-Token": api_key},
+                    timeout=10.0,
                 )
                 r.raise_for_status()
-
-            results = r.json().get("web", {}).get("results", [])[:n]
-            if not results:
-                return f"No results for: {query}"
-
-            lines = [f"Results for: {query}\n"]
-            for i, item in enumerate(results, 1):
-                lines.append(f"{i}. {item.get('title', '')}\n   {item.get('url', '')}")
-                if desc := item.get("description"):
-                    lines.append(f"   {desc}")
-            return "\n".join(lines)
-        except httpx.ProxyError as e:
-            logger.error("WebSearch proxy error: {}", e)
-            return f"Proxy error: {e}"
+            items = [
+                {"title": x.get("title", ""), "url": x.get("url", ""), "content": x.get("description", "")}
+                for x in r.json().get("web", {}).get("results", [])
+            ]
+            return _format_results(query, items, n)
         except Exception as e:
-            logger.error("WebSearch error: {}", e)
             return f"Error: {e}"
 
+    async def _search_tavily(self, query: str, n: int) -> str:
+        api_key = self.config.api_key or os.environ.get("TAVILY_API_KEY", "")
+        if not api_key:
+            logger.warning("TAVILY_API_KEY not set, falling back to DuckDuckGo")
+            return await self._search_duckduckgo(query, n)
+        try:
+            async with httpx.AsyncClient(proxy=self.proxy) as client:
+                r = await client.post(
+                    "https://api.tavily.com/search",
+                    headers={"Authorization": f"Bearer {api_key}"},
+                    json={"query": query, "max_results": n},
+                    timeout=15.0,
+                )
+                r.raise_for_status()
+            return _format_results(query, r.json().get("results", []), n)
+        except Exception as e:
+            return f"Error: {e}"
+
+    async def _search_searxng(self, query: str, n: int) -> str:
+        base_url = (self.config.base_url or os.environ.get("SEARXNG_BASE_URL", "")).strip()
+        if not base_url:
+            logger.warning("SEARXNG_BASE_URL not set, falling back to DuckDuckGo")
+            return await self._search_duckduckgo(query, n)
+        endpoint = f"{base_url.rstrip('/')}/search"
+        is_valid, error_msg = _validate_url(endpoint)
+        if not is_valid:
+            return f"Error: invalid SearXNG URL: {error_msg}"
+        try:
+            async with httpx.AsyncClient(proxy=self.proxy) as client:
+                r = await client.get(
+                    endpoint,
+                    params={"q": query, "format": "json"},
+                    headers={"User-Agent": USER_AGENT},
+                    timeout=10.0,
+                )
+                r.raise_for_status()
+            return _format_results(query, r.json().get("results", []), n)
+        except Exception as e:
+            return f"Error: {e}"
+
+    async def _search_jina(self, query: str, n: int) -> str:
+        api_key = self.config.api_key or os.environ.get("JINA_API_KEY", "")
+        if not api_key:
+            logger.warning("JINA_API_KEY not set, falling back to DuckDuckGo")
+            return await self._search_duckduckgo(query, n)
+        try:
+            headers = {"Accept": "application/json", "Authorization": f"Bearer {api_key}"}
+            async with httpx.AsyncClient(proxy=self.proxy) as client:
+                r = await client.get(
+                    f"https://s.jina.ai/",
+                    params={"q": query},
+                    headers=headers,
+                    timeout=15.0,
+                )
+                r.raise_for_status()
+            data = r.json().get("data", [])[:n]
+            items = [
+                {"title": d.get("title", ""), "url": d.get("url", ""), "content": d.get("content", "")[:500]}
+                for d in data
+            ]
+            return _format_results(query, items, n)
+        except Exception as e:
+            return f"Error: {e}"
+
+    async def _search_duckduckgo(self, query: str, n: int) -> str:
+        try:
+            from ddgs import DDGS
+
+            ddgs = DDGS(timeout=10)
+            raw = await asyncio.to_thread(ddgs.text, query, max_results=n)
+            if not raw:
+                return f"No results for: {query}"
+            items = [
+                {"title": r.get("title", ""), "url": r.get("href", ""), "content": r.get("body", "")}
+                for r in raw
+            ]
+            return _format_results(query, items, n)
+        except Exception as e:
+            logger.warning("DuckDuckGo search failed: {}", e)
+            return f"Error: DuckDuckGo search failed ({e})"
+
 
 class WebFetchTool(Tool):
-    """Fetch and extract content from a URL using Readability."""
+    """Fetch and extract content from a URL."""
 
     name = "web_fetch"
     description = "Fetch URL and extract readable content (HTML → markdown/text)."
@@ -116,9 +215,9 @@ class WebFetchTool(Tool):
         "properties": {
             "url": {"type": "string", "description": "URL to fetch"},
             "extractMode": {"type": "string", "enum": ["markdown", "text"], "default": "markdown"},
-            "maxChars": {"type": "integer", "minimum": 100}
+            "maxChars": {"type": "integer", "minimum": 100},
         },
-        "required": ["url"]
+        "required": ["url"],
     }
 
     def __init__(self, max_chars: int = 50000, proxy: str | None = None):
@@ -126,15 +225,55 @@ class WebFetchTool(Tool):
         self.proxy = proxy
 
     async def execute(self, url: str, extractMode: str = "markdown", maxChars: int | None = None, **kwargs: Any) -> str:
-        from readability import Document
-
         max_chars = maxChars or self.max_chars
         is_valid, error_msg = _validate_url(url)
         if not is_valid:
             return json.dumps({"error": f"URL validation failed: {error_msg}", "url": url}, ensure_ascii=False)
 
+        result = await self._fetch_jina(url, max_chars)
+        if result is None:
+            result = await self._fetch_readability(url, extractMode, max_chars)
+        return result
+
+    async def _fetch_jina(self, url: str, max_chars: int) -> str | None:
+        """Try fetching via Jina Reader API. Returns None on failure."""
+        try:
+            headers = {"Accept": "application/json", "User-Agent": USER_AGENT}
+            jina_key = os.environ.get("JINA_API_KEY", "")
+            if jina_key:
+                headers["Authorization"] = f"Bearer {jina_key}"
+            async with httpx.AsyncClient(proxy=self.proxy, timeout=20.0) as client:
+                r = await client.get(f"https://r.jina.ai/{url}", headers=headers)
+                if r.status_code == 429:
+                    logger.debug("Jina Reader rate limited, falling back to readability")
+                    return None
+                r.raise_for_status()
+
+            data = r.json().get("data", {})
+            title = data.get("title", "")
+            text = data.get("content", "")
+            if not text:
+                return None
+
+            if title:
+                text = f"# {title}\n\n{text}"
+            truncated = len(text) > max_chars
+            if truncated:
+                text = text[:max_chars]
+
+            return json.dumps({
+                "url": url, "finalUrl": data.get("url", url), "status": r.status_code,
+                "extractor": "jina", "truncated": truncated, "length": len(text), "text": text,
+            }, ensure_ascii=False)
+        except Exception as e:
+            logger.debug("Jina Reader failed for {}, falling back to readability: {}", url, e)
+            return None
+
+    async def _fetch_readability(self, url: str, extract_mode: str, max_chars: int) -> str:
+        """Local fallback using readability-lxml."""
+        from readability import Document
+
         try:
-            logger.debug("WebFetch: {}", "proxy enabled" if self.proxy else "direct connection")
             async with httpx.AsyncClient(
                 follow_redirects=True,
                 max_redirects=MAX_REDIRECTS,
@@ -150,17 +289,20 @@ class WebFetchTool(Tool):
                 text, extractor = json.dumps(r.json(), indent=2, ensure_ascii=False), "json"
             elif "text/html" in ctype or r.text[:256].lower().startswith(("<!doctype", "<html")):
                 doc = Document(r.text)
-                content = self._to_markdown(doc.summary()) if extractMode == "markdown" else _strip_tags(doc.summary())
+                content = self._to_markdown(doc.summary()) if extract_mode == "markdown" else _strip_tags(doc.summary())
                 text = f"# {doc.title()}\n\n{content}" if doc.title() else content
                 extractor = "readability"
             else:
                 text, extractor = r.text, "raw"
 
             truncated = len(text) > max_chars
-            if truncated: text = text[:max_chars]
+            if truncated:
+                text = text[:max_chars]
 
-            return json.dumps({"url": url, "finalUrl": str(r.url), "status": r.status_code,
-                              "extractor": extractor, "truncated": truncated, "length": len(text), "text": text}, ensure_ascii=False)
+            return json.dumps({
+                "url": url, "finalUrl": str(r.url), "status": r.status_code,
+                "extractor": extractor, "truncated": truncated, "length": len(text), "text": text,
+            }, ensure_ascii=False)
         except httpx.ProxyError as e:
             logger.error("WebFetch proxy error for {}: {}", url, e)
             return json.dumps({"error": f"Proxy error: {e}", "url": url}, ensure_ascii=False)
@@ -168,11 +310,10 @@ class WebFetchTool(Tool):
             logger.error("WebFetch error for {}: {}", url, e)
             return json.dumps({"error": str(e), "url": url}, ensure_ascii=False)
 
-    def _to_markdown(self, html: str) -> str:
+    def _to_markdown(self, html_content: str) -> str:
         """Convert HTML to markdown."""
-        # Convert links, headings, lists before stripping tags
         text = re.sub(r'<a\s+[^>]*href=["\']([^"\']+)["\'][^>]*>([\s\S]*?)</a>',
-                      lambda m: f'[{_strip_tags(m[2])}]({m[1]})', html, flags=re.I)
+                      lambda m: f'[{_strip_tags(m[2])}]({m[1]})', html_content, flags=re.I)
         text = re.sub(r'<h([1-6])[^>]*>([\s\S]*?)</h\1>',
                       lambda m: f'\n{"#" * int(m[1])} {_strip_tags(m[2])}\n', text, flags=re.I)
         text = re.sub(r'<li[^>]*>([\s\S]*?)</li>', lambda m: f'\n- {_strip_tags(m[1])}', text, flags=re.I)