feat(web-tools): add option to disable fetching via Jina Reader

A new configuration block has been added for the web fetch tool, which
allows forcing the tool to use the local readability-lxml mode.

Combined with the previous option to modify the user agent, allows
bypassing most Cloudflare captchas and JS proof-of-work.

Assisted-by: Jo'Zahir:Qwen3.6-35B-A3B

nanobot/agent/loop.py

@@ -292,6 +292,7 @@ class AgentLoop:
             )
             self.tools.register(
                 WebFetchTool(
+                    config=self.web_config.fetch,
                     proxy=self.web_config.proxy,
                     user_agent=self.web_config.user_agent,
                 )

nanobot/agent/subagent.py

@@ -182,6 +182,7 @@ class SubagentManager:
                 )
                 tools.register(
                     WebFetchTool(
+                        config=self.web_config.fetch,
                         proxy=self.web_config.proxy,
                         user_agent=self.web_config.user_agent,
                     )

nanobot/agent/tools/web.py

@@ -18,7 +18,7 @@ from nanobot.agent.tools.schema import IntegerSchema, StringSchema, tool_paramet
 from nanobot.utils.helpers import build_image_content_blocks
 
 if TYPE_CHECKING:
-    from nanobot.config.schema import WebSearchConfig
+    from nanobot.config.schema import WebSearchConfig, WebFetchConfig
 
 # Shared constants
 _DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36"
@@ -304,10 +304,13 @@ class WebFetchTool(Tool):
         "Works for most web pages and docs; may fail on login-walled or JS-heavy sites."
     )
 
-    def __init__(self, max_chars: int = 50000, proxy: str | None = None, user_agent: str | None = None):
-        self.max_chars = max_chars
+    def __init__(self, config: WebFetchConfig | None = None, proxy: str | None = None, user_agent: str | None = None, max_chars: int = 50000):
+        from nanobot.config.schema import WebFetchConfig
+
+        self.config = config if config is not None else WebFetchConfig()
         self.proxy = proxy
         self.user_agent = user_agent or _DEFAULT_USER_AGENT
+        self.max_chars = max_chars
 
     @property
     def read_only(self) -> bool:
@@ -337,7 +340,9 @@ class WebFetchTool(Tool):
         except Exception as e:
             logger.debug("Pre-fetch image detection failed for {}: {}", url, e)
 
-        result = await self._fetch_jina(url, max_chars)
+        result = None
+        if self.config.use_jina_reader:
+            result = await self._fetch_jina(url, max_chars)
         if result is None:
             result = await self._fetch_readability(url, extractMode, max_chars)
         return result

nanobot/config/schema.py

@@ -175,6 +175,12 @@ class WebSearchConfig(Base):
     timeout: int = 30  # Wall-clock timeout (seconds) for search operations
 
 
+class WebFetchConfig(Base):
+    """Web fetch tool configuration."""
+
+    use_jina_reader: bool = True
+
+
 class WebToolsConfig(Base):
     """Web tools configuration."""
 
@@ -184,6 +190,7 @@ class WebToolsConfig(Base):
     )
     user_agent: str | None = None
     search: WebSearchConfig = Field(default_factory=WebSearchConfig)
+    fetch: WebFetchConfig = Field(default_factory=WebFetchConfig)
 
 
 class ExecToolConfig(Base):