feat(web-search): add Serper.dev (Google Search API) provider
Add 'serper' as a web search backend, following the existing provider pattern (keenable/exa): POST to https://google.serper.dev/search with the X-API-KEY header, map the 'organic' results into the shared result format, and fall back to DuckDuckGo when no key is configured. - key resolved from config.api_key or SERPER_API_KEY env var - 429 handled with a rate-limit message; other HTTP errors surfaced - tests cover success, env-key, no-key fallback, HTTP error and rate limit - docs: add Serper config example and list it in tools.web.search providers
This commit is contained in:
committed by
Xubin Ren
parent
72c8a47ac9
commit
8a42a9c73a
+17
-1
@@ -1750,6 +1750,22 @@ You can also set `WEB_SEARCH_API_KEY` for compatibility with the Volcengine web-
|
|||||||
|
|
||||||
Keenable search works out of the box with no account, via its token-less public endpoint (free tier, limited to 1,000 requests/hour). Set `apiKey` (or `KEENABLE_API_KEY`) from [keenable.ai](https://keenable.ai) to remove the hourly limit.
|
Keenable search works out of the box with no account, via its token-less public endpoint (free tier, limited to 1,000 requests/hour). Set `apiKey` (or `KEENABLE_API_KEY`) from [keenable.ai](https://keenable.ai) to remove the hourly limit.
|
||||||
|
|
||||||
|
**Serper** (Google Search API):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"tools": {
|
||||||
|
"web": {
|
||||||
|
"search": {
|
||||||
|
"provider": "serper",
|
||||||
|
"apiKey": "${SERPER_API_KEY}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a key at [serper.dev](https://serper.dev). You can also set `SERPER_API_KEY` in the environment instead of storing it in config.
|
||||||
|
|
||||||
**SearXNG** (self-hosted, no API key needed):
|
**SearXNG** (self-hosted, no API key needed):
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -1781,7 +1797,7 @@ Keenable search works out of the box with no account, via its token-less public
|
|||||||
|
|
||||||
| Option | Type | Default | Description |
|
| Option | Type | Default | Description |
|
||||||
|--------|------|---------|-------------|
|
|--------|------|---------|-------------|
|
||||||
| `provider` | string | `"duckduckgo"` | Search backend: `brave`, `tavily`, `jina`, `kagi`, `olostep`, `bocha`, `volcengine`, `keenable`, `searxng`, `duckduckgo` |
|
| `provider` | string | `"duckduckgo"` | Search backend: `brave`, `tavily`, `jina`, `kagi`, `olostep`, `bocha`, `volcengine`, `keenable`, `serper`, `searxng`, `duckduckgo` |
|
||||||
| `apiKey` | string | `""` | API key for API-backed search providers |
|
| `apiKey` | string | `""` | API key for API-backed search providers |
|
||||||
| `baseUrl` | string | `""` | Base URL for SearXNG |
|
| `baseUrl` | string | `""` | Base URL for SearXNG |
|
||||||
| `maxResults` | integer | `5` | Results per search (1–10) |
|
| `maxResults` | integer | `5` | Results per search (1–10) |
|
||||||
|
|||||||
@@ -338,6 +338,9 @@ class WebSearchTool(Tool):
|
|||||||
return "volcengine" if api_key else "duckduckgo"
|
return "volcengine" if api_key else "duckduckgo"
|
||||||
if provider == "keenable":
|
if provider == "keenable":
|
||||||
return "keenable"
|
return "keenable"
|
||||||
|
if provider == "serper":
|
||||||
|
api_key = self.config.api_key or os.environ.get("SERPER_API_KEY", "")
|
||||||
|
return "serper" if api_key else "duckduckgo"
|
||||||
return provider
|
return provider
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -394,6 +397,8 @@ class WebSearchTool(Tool):
|
|||||||
)
|
)
|
||||||
elif provider == "keenable":
|
elif provider == "keenable":
|
||||||
return await self._search_keenable(query, n)
|
return await self._search_keenable(query, n)
|
||||||
|
elif provider == "serper":
|
||||||
|
return await self._search_serper(query, n)
|
||||||
else:
|
else:
|
||||||
return ToolResult.error(f"Error: unknown search provider '{provider}'")
|
return ToolResult.error(f"Error: unknown search provider '{provider}'")
|
||||||
|
|
||||||
@@ -668,6 +673,43 @@ class WebSearchTool(Tool):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
return ToolResult.error(f"Error: Exa search failed: {e}")
|
return ToolResult.error(f"Error: Exa search failed: {e}")
|
||||||
|
|
||||||
|
async def _search_serper(self, query: str, n: int) -> str:
|
||||||
|
"""Search via Serper.dev (Google Search API)."""
|
||||||
|
api_key = self.config.api_key or os.environ.get("SERPER_API_KEY", "")
|
||||||
|
if not api_key:
|
||||||
|
logger.warning("SERPER_API_KEY not set, falling back to DuckDuckGo")
|
||||||
|
return await self._search_duckduckgo(query, n)
|
||||||
|
try:
|
||||||
|
headers = {
|
||||||
|
"X-API-KEY": api_key,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"User-Agent": self.user_agent,
|
||||||
|
}
|
||||||
|
async with httpx.AsyncClient(proxy=self.proxy) as client:
|
||||||
|
r = await client.post(
|
||||||
|
"https://google.serper.dev/search",
|
||||||
|
headers=headers,
|
||||||
|
json={"q": query, "num": n},
|
||||||
|
timeout=float(self.config.timeout),
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
items = [
|
||||||
|
{
|
||||||
|
"title": result.get("title", ""),
|
||||||
|
"url": result.get("link", ""),
|
||||||
|
"content": result.get("snippet", ""),
|
||||||
|
}
|
||||||
|
for result in r.json().get("organic", [])
|
||||||
|
if isinstance(result, dict)
|
||||||
|
]
|
||||||
|
return _format_results(query, items, n)
|
||||||
|
except httpx.HTTPStatusError as e:
|
||||||
|
if e.response.status_code == 429:
|
||||||
|
return "Error: Serper search rate limited. Try again later or reduce search frequency."
|
||||||
|
return f"Error: Serper search failed ({e.response.status_code}): {e}"
|
||||||
|
except Exception as e:
|
||||||
|
return f"Error: Serper search failed: {e}"
|
||||||
|
|
||||||
async def _search_volcengine(
|
async def _search_volcengine(
|
||||||
self,
|
self,
|
||||||
query: str,
|
query: str,
|
||||||
|
|||||||
@@ -201,6 +201,89 @@ async def test_keenable_search_http_error(monkeypatch):
|
|||||||
assert "Error: Keenable search failed (401)" in result
|
assert "Error: Keenable search failed (401)" in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_serper_without_api_key_is_treated_as_duckduckgo(monkeypatch):
|
||||||
|
# Serper requires a key; without one we fall back to DuckDuckGo for concurrency.
|
||||||
|
monkeypatch.delenv("SERPER_API_KEY", raising=False)
|
||||||
|
tool = _tool(provider="serper", api_key="")
|
||||||
|
assert tool.exclusive is True
|
||||||
|
assert tool.concurrency_safe is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_serper_search(monkeypatch):
|
||||||
|
async def mock_post(self, url, **kw):
|
||||||
|
assert url == "https://google.serper.dev/search"
|
||||||
|
assert kw["headers"]["X-API-KEY"] == "serper-key"
|
||||||
|
assert kw["headers"]["User-Agent"] == "nanobot-search-test"
|
||||||
|
assert kw["json"] == {"q": "serper", "num": 1}
|
||||||
|
return _response(json={
|
||||||
|
"organic": [
|
||||||
|
{"title": "Serper", "link": "https://serper.dev", "snippet": "Google Search API"}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
|
||||||
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
||||||
|
tool = _tool(provider="serper", api_key="serper-key", user_agent="nanobot-search-test")
|
||||||
|
result = await tool.execute(query="serper", count=1)
|
||||||
|
assert "Serper" in result
|
||||||
|
assert "https://serper.dev" in result
|
||||||
|
assert "Google Search API" in result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_serper_search_uses_env_api_key(monkeypatch):
|
||||||
|
async def mock_post(self, url, **kw):
|
||||||
|
assert kw["headers"]["X-API-KEY"] == "env-serper-key"
|
||||||
|
return _response(json={
|
||||||
|
"organic": [{"title": "Env", "link": "https://serper.dev/env", "snippet": "ok"}]
|
||||||
|
})
|
||||||
|
|
||||||
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
||||||
|
monkeypatch.setenv("SERPER_API_KEY", "env-serper-key")
|
||||||
|
tool = _tool(provider="serper", api_key="")
|
||||||
|
result = await tool.execute(query="serper", count=1)
|
||||||
|
assert "Env" in result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_serper_fallback_to_duckduckgo_when_no_key(monkeypatch):
|
||||||
|
class MockDDGS:
|
||||||
|
def __init__(self, **kw):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def text(self, query, max_results=5):
|
||||||
|
return [{"title": "Fallback", "href": "https://ddg.example", "body": "DuckDuckGo fallback"}]
|
||||||
|
|
||||||
|
monkeypatch.setattr("ddgs.DDGS", MockDDGS)
|
||||||
|
monkeypatch.delenv("SERPER_API_KEY", raising=False)
|
||||||
|
|
||||||
|
tool = _tool(provider="serper", api_key="")
|
||||||
|
result = await tool.execute(query="serper", count=1)
|
||||||
|
assert "DuckDuckGo fallback" in result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_serper_search_http_error(monkeypatch):
|
||||||
|
async def mock_post(self, url, **kw):
|
||||||
|
return _response(status=403, json={"message": "Unauthorized"})
|
||||||
|
|
||||||
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
||||||
|
tool = _tool(provider="serper", api_key="bad-serper-key")
|
||||||
|
result = await tool.execute(query="serper")
|
||||||
|
assert "Error: Serper search failed (403)" in result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_serper_search_rate_limited(monkeypatch):
|
||||||
|
async def mock_post(self, url, **kw):
|
||||||
|
return _response(status=429, json={"message": "rate limited"})
|
||||||
|
|
||||||
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
||||||
|
tool = _tool(provider="serper", api_key="serper-key")
|
||||||
|
result = await tool.execute(query="serper")
|
||||||
|
assert "Serper search rate limited" in result
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_bocha_search(monkeypatch):
|
async def test_bocha_search(monkeypatch):
|
||||||
async def mock_post(self, url, **kw):
|
async def mock_post(self, url, **kw):
|
||||||
|
|||||||
Reference in New Issue
Block a user