feat: implement native multimodal autonomous sensory capabilities

nanobot/agent/tools/base.py

@@ -21,6 +21,20 @@ class Tool(ABC):
         "object": dict,
     }
 
+    @staticmethod
+    def _resolve_type(t: Any) -> str | None:
+        """Resolve JSON Schema type to a simple string.
+
+        JSON Schema allows ``"type": ["string", "null"]`` (union types).
+        We extract the first non-null type so validation/casting works.
+        """
+        if isinstance(t, list):
+            for item in t:
+                if item != "null":
+                    return item
+            return None
+        return t
+
     @property
     @abstractmethod
     def name(self) -> str:
@@ -40,7 +54,7 @@ class Tool(ABC):
         pass
 
     @abstractmethod
-    async def execute(self, **kwargs: Any) -> str:
+    async def execute(self, **kwargs: Any) -> Any:
         """
         Execute the tool with given parameters.
 
@@ -48,7 +62,7 @@ class Tool(ABC):
             **kwargs: Tool-specific parameters.
 
         Returns:
-            String result of the tool execution.
+            Result of the tool execution (string or list of content blocks).
         """
         pass
 
@@ -78,7 +92,7 @@ class Tool(ABC):
 
     def _cast_value(self, val: Any, schema: dict[str, Any]) -> Any:
         """Cast a single value according to schema."""
-        target_type = schema.get("type")
+        target_type = self._resolve_type(schema.get("type"))
 
         if target_type == "boolean" and isinstance(val, bool):
             return val
@@ -131,7 +145,11 @@ class Tool(ABC):
         return self._validate(params, {**schema, "type": "object"}, "")
 
     def _validate(self, val: Any, schema: dict[str, Any], path: str) -> list[str]:
-        t, label = schema.get("type"), path or "parameter"
+        raw_type = schema.get("type")
+        nullable = isinstance(raw_type, list) and "null" in raw_type
+        t, label = self._resolve_type(raw_type), path or "parameter"
+        if nullable and val is None:
+            return []
         if t == "integer" and (not isinstance(val, int) or isinstance(val, bool)):
             return [f"{label} should be integer"]
         if t == "number" and (

nanobot/agent/tools/filesystem.py

@@ -1,10 +1,13 @@
 """File system tools: read, write, edit, list."""
 
+import base64
 import difflib
+import mimetypes
 from pathlib import Path
 from typing import Any
 
 from nanobot.agent.tools.base import Tool
+from nanobot.utils.helpers import detect_image_mime
 
 
 def _resolve_path(
@@ -91,7 +94,7 @@ class ReadFileTool(_FsTool):
             "required": ["path"],
         }
 
-    async def execute(self, path: str, offset: int = 1, limit: int | None = None, **kwargs: Any) -> str:
+    async def execute(self, path: str, offset: int = 1, limit: int | None = None, **kwargs: Any) -> Any:
         try:
             fp = self._resolve(path)
             if not fp.exists():
@@ -99,13 +102,28 @@ class ReadFileTool(_FsTool):
             if not fp.is_file():
                 return f"Error: Not a file: {path}"
 
-            all_lines = fp.read_text(encoding="utf-8").splitlines()
+            raw = fp.read_bytes()
+            if not raw:
+                return f"(Empty file: {path})"
+
+            mime = detect_image_mime(raw) or mimetypes.guess_type(path)[0]
+            if mime and mime.startswith("image/"):
+                b64 = base64.b64encode(raw).decode()
+                return [
+                    {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{b64}"}, "_meta": {"path": str(fp)}},
+                    {"type": "text", "text": f"(Image file: {path})"}
+                ]
+
+            try:
+                text_content = raw.decode("utf-8")
+            except UnicodeDecodeError:
+                return f"Error: Cannot read binary file {path} (MIME: {mime or 'unknown'}). Only UTF-8 text and images are supported."
+
+            all_lines = text_content.splitlines()
             total = len(all_lines)
 
             if offset < 1:
                 offset = 1
-            if total == 0:
-                return f"(Empty file: {path})"
             if offset > total:
                 return f"Error: offset {offset} is beyond end of file ({total} lines)"
 

nanobot/agent/tools/registry.py

@@ -35,7 +35,7 @@ class ToolRegistry:
         """Get all tool definitions in OpenAI format."""
         return [tool.to_schema() for tool in self._tools.values()]
 
-    async def execute(self, name: str, params: dict[str, Any]) -> str:
+    async def execute(self, name: str, params: dict[str, Any]) -> Any:
         """Execute a tool by name with given parameters."""
         _HINT = "\n\n[Analyze the error above and try a different approach.]"
 

nanobot/agent/tools/web.py

@@ -3,8 +3,10 @@
 from __future__ import annotations
 
 import asyncio
+import base64
 import html
 import json
+import mimetypes
 import os
 import re
 from typing import TYPE_CHECKING, Any
@@ -196,6 +198,8 @@ class WebSearchTool(Tool):
 
     async def _search_duckduckgo(self, query: str, n: int) -> str:
         try:
+            # Note: duckduckgo_search is synchronous and does its own requests
+            # We run it in a thread to avoid blocking the loop
             from ddgs import DDGS
 
             ddgs = DDGS(timeout=10)
@@ -231,12 +235,28 @@ class WebFetchTool(Tool):
         self.max_chars = max_chars
         self.proxy = proxy
 
-    async def execute(self, url: str, extractMode: str = "markdown", maxChars: int | None = None, **kwargs: Any) -> str:
+    async def execute(self, url: str, extractMode: str = "markdown", maxChars: int | None = None, **kwargs: Any) -> Any:
         max_chars = maxChars or self.max_chars
         is_valid, error_msg = _validate_url_safe(url)
         if not is_valid:
             return json.dumps({"error": f"URL validation failed: {error_msg}", "url": url}, ensure_ascii=False)
 
+        # Detect and fetch images directly to avoid Jina's textual image captioning
+        try:
+            async with httpx.AsyncClient(proxy=self.proxy, follow_redirects=True, max_redirects=MAX_REDIRECTS, timeout=15.0) as client:
+                async with client.stream("GET", url, headers={"User-Agent": USER_AGENT}) as r:
+                    ctype = r.headers.get("content-type", "")
+                    if ctype.startswith("image/"):
+                        await r.aread()
+                        r.raise_for_status()
+                        b64 = base64.b64encode(r.content).decode()
+                        return [
+                            {"type": "image_url", "image_url": {"url": f"data:{ctype};base64,{b64}"}, "_meta": {"path": url}},
+                            {"type": "text", "text": f"(Image fetched from: {url})"}
+                        ]
+        except Exception as e:
+            logger.debug("Pre-fetch image detection failed for {}: {}", url, e)
+
         result = await self._fetch_jina(url, max_chars)
         if result is None:
             result = await self._fetch_readability(url, extractMode, max_chars)
@@ -278,7 +298,7 @@ class WebFetchTool(Tool):
             logger.debug("Jina Reader failed for {}, falling back to readability: {}", url, e)
             return None
 
-    async def _fetch_readability(self, url: str, extract_mode: str, max_chars: int) -> str:
+    async def _fetch_readability(self, url: str, extract_mode: str, max_chars: int) -> Any:
         """Local fallback using readability-lxml."""
         from readability import Document
 
@@ -298,6 +318,12 @@ class WebFetchTool(Tool):
                 return json.dumps({"error": f"Redirect blocked: {redir_err}", "url": url}, ensure_ascii=False)
 
             ctype = r.headers.get("content-type", "")
+            if ctype.startswith("image/"):
+                b64 = base64.b64encode(r.content).decode()
+                return [
+                    {"type": "image_url", "image_url": {"url": f"data:{ctype};base64,{b64}"}, "_meta": {"path": url}},
+                    {"type": "text", "text": f"(Image fetched from: {url})"}
+                ]
 
             if "application/json" in ctype:
                 text, extractor = json.dumps(r.json(), indent=2, ensure_ascii=False), "json"