Grayscale/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(api): support file uploads via JSON base64 and multipart/form-data

Browse Source
This commit is contained in:
dengjingren
2026-04-08 15:58:52 +08:00
parent e21ba5f667
commit a068df5a79
10 changed files with 1188 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nanobot/agent/context.py
+39 -14
View File
@@ -144,31 +144,56 @@ class ContextBuilder:
messages.append({"role": current_role, "content": merged})
return messages
def _build_user_content(self, text: str, media: list[str] | None) -> str | list[dict[str, Any]]:
"""Build user message content with optional base64-encoded images."""
def _build_user_content(
self, text: str, media: list[str] | None
) -> str | list[dict[str, Any]]:
"""Build user message content with optional media.
Images are converted to base64 vision blocks.
Documents (PDF, Word, Excel, PPT) have their text extracted and appended.
"""
if not media:
return text
images = []
images: list[dict[str, Any]] = []
doc_texts: list[str] = []
for path in media:
p = Path(path)
if not p.is_file():
continue
raw = p.read_bytes()
# Detect real MIME type from magic bytes; fallback to filename guess
mime = detect_image_mime(raw) or mimetypes.guess_type(path)[0]
if not mime or not mime.startswith("image/"):
continue
b64 = base64.b64encode(raw).decode()
images.append({
"type": "image_url",
"image_url": {"url": f"data:{mime};base64,{b64}"},
"_meta": {"path": str(p)},
})
if not images:
if mime and mime.startswith("image/"):
b64 = base64.b64encode(raw).decode()
images.append({
"type": "image_url",
"image_url": {"url": f"data:{mime};base64,{b64}"},
"_meta": {"path": str(p)},
})
else:
# Try document text extraction
from nanobot.utils.document import extract_text
extracted = extract_text(p)
if extracted and not extracted.startswith("Error"):
doc_texts.append(f"[File: {p.name}]\n{extracted}")
# Build final content
parts: list[dict[str, Any]] = []
parts.extend(images)
combined_text = text
if doc_texts:
combined_text = text + "\n\n" + "\n\n".join(doc_texts)
if images:
parts.append({"type": "text", "text": combined_text})
return parts
elif doc_texts:
return combined_text
else:
return text
return images + [{"type": "text", "text": text}]
def add_tool_result(
self, messages: list[dict[str, Any]],
nanobot/agent/loop.py
+5 -1
View File
@@ -765,13 +765,17 @@ class AgentLoop:
session_key: str = "cli:direct",
channel: str = "cli",
chat_id: str = "direct",
media: list[str] | None = None,
on_progress: Callable[[str], Awaitable[None]] | None = None,
on_stream: Callable[[str], Awaitable[None]] | None = None,
on_stream_end: Callable[..., Awaitable[None]] | None = None,
) -> OutboundMessage | None:
"""Process a message directly and return the outbound payload."""
await self._connect_mcp()
msg = InboundMessage(channel=channel, sender_id="user", chat_id=chat_id, content=content)
msg = InboundMessage(
channel=channel, sender_id="user", chat_id=chat_id,
content=content, media=media or [],
)
return await self._process_message(
msg, session_key=session_key, on_progress=on_progress,
on_stream=on_stream, on_stream_end=on_stream_end,
nanobot/api/server.py
+135 -40
View File
@@ -7,15 +7,28 @@ All requests route to a single persistent API session.
from __future__ import annotations
import asyncio
import base64
import mimetypes
import re
import time
import uuid
from pathlib import Path
from typing import Any
from aiohttp import web
from loguru import logger
from nanobot.config.paths import get_media_dir
from nanobot.utils.helpers import safe_filename
from nanobot.utils.runtime import EMPTY_FINAL_RESPONSE_MESSAGE
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MB
_DATA_URL_RE = re.compile(r"^data:([^;]+);base64,(.+)$", re.DOTALL)
class _FileSizeExceeded(Exception):
"""Raised when an uploaded file exceeds the size limit."""
API_SESSION_KEY = "api:default"
API_CHAT_ID = "default"
@@ -57,48 +70,134 @@ def _response_text(value: Any) -> str:
return str(value)
# ---------------------------------------------------------------------------
# Upload helpers
# ---------------------------------------------------------------------------
def _save_base64_data_url(data_url: str, media_dir: Path) -> str | None:
"""Decode a data:...;base64,... URL and save to disk."""
m = _DATA_URL_RE.match(data_url)
if not m:
return None
mime_type, b64_payload = m.group(1), m.group(2)
try:
raw = base64.b64decode(b64_payload)
except Exception:
return None
ext = mimetypes.guess_extension(mime_type) or ".bin"
filename = f"{uuid.uuid4().hex[:12]}{ext}"
dest = media_dir / safe_filename(filename)
dest.write_bytes(raw)
return str(dest)
def _parse_json_content(body: dict) -> tuple[str, list[str]]:
"""Parse JSON request body. Returns (text, media_paths)."""
messages = body.get("messages")
if not isinstance(messages, list) or len(messages) != 1:
raise ValueError("Only a single user message is supported")
message = messages[0]
if not isinstance(message, dict) or message.get("role") != "user":
raise ValueError("Only a single user message is supported")
user_content = message.get("content", "")
media_dir = get_media_dir("api")
media_paths: list[str] = []
if isinstance(user_content, list):
text_parts: list[str] = []
for part in user_content:
if not isinstance(part, dict):
continue
if part.get("type") == "text":
text_parts.append(part.get("text", ""))
elif part.get("type") == "image_url":
url = part.get("image_url", {}).get("url", "")
if url.startswith("data:"):
saved = _save_base64_data_url(url, media_dir)
if saved:
media_paths.append(saved)
text = " ".join(text_parts)
elif isinstance(user_content, str):
text = user_content
else:
raise ValueError("Invalid content format")
return text, media_paths
async def _parse_multipart(request: web.Request) -> tuple[str, list[str], str | None]:
"""Parse multipart/form-data. Returns (text, media_paths, session_id)."""
media_dir = get_media_dir("api")
reader = await request.multipart()
text = ""
session_id = None
media_paths: list[str] = []
while True:
part = await reader.next()
if part is None:
break
if part.name == "message":
text = (await part.read()).decode("utf-8")
elif part.name == "session_id":
session_id = (await part.read()).decode("utf-8").strip()
elif part.name == "files":
raw = await part.read()
if len(raw) > MAX_FILE_SIZE:
raise _FileSizeExceeded(f"File '{part.filename}' exceeds {MAX_FILE_SIZE // (1024*1024)}MB limit")
filename = safe_filename(part.filename or f"{uuid.uuid4().hex[:12]}.bin")
dest = media_dir / filename
dest.write_bytes(raw)
media_paths.append(str(dest))
if not text:
text = "请分析上传的文件"
return text, media_paths, session_id
# ---------------------------------------------------------------------------
# Route handlers
# ---------------------------------------------------------------------------
async def handle_chat_completions(request: web.Request) -> web.Response:
"""POST /v1/chat/completions"""
# --- Parse body ---
try:
body = await request.json()
except Exception:
return _error_json(400, "Invalid JSON body")
messages = body.get("messages")
if not isinstance(messages, list) or len(messages) != 1:
return _error_json(400, "Only a single user message is supported")
# Stream not yet supported
if body.get("stream", False):
return _error_json(400, "stream=true is not supported yet. Set stream=false or omit it.")
message = messages[0]
if not isinstance(message, dict) or message.get("role") != "user":
return _error_json(400, "Only a single user message is supported")
user_content = message.get("content", "")
if isinstance(user_content, list):
# Multi-modal content array — extract text parts
user_content = " ".join(
part.get("text", "") for part in user_content if part.get("type") == "text"
)
"""POST /v1/chat/completions — supports JSON and multipart/form-data."""
content_type = request.content_type or ""
if not isinstance(content_type, str):
content_type = ""
agent_loop = request.app["agent_loop"]
timeout_s: float = request.app.get("request_timeout", 120.0)
model_name: str = request.app.get("model_name", "nanobot")
if (requested_model := body.get("model")) and requested_model != model_name:
return _error_json(400, f"Only configured model '{model_name}' is available")
session_key = f"api:{body['session_id']}" if body.get("session_id") else API_SESSION_KEY
try:
if content_type.startswith("multipart/"):
text, media_paths, session_id = await _parse_multipart(request)
else:
try:
body = await request.json()
except Exception:
return _error_json(400, "Invalid JSON body")
if body.get("stream", False):
return _error_json(400, "stream=true is not supported yet. Set stream=false or omit it.")
if (requested_model := body.get("model")) and requested_model != model_name:
return _error_json(400, f"Only configured model '{model_name}' is available")
text, media_paths = _parse_json_content(body)
session_id = body.get("session_id")
except ValueError as e:
return _error_json(400, str(e))
except _FileSizeExceeded as e:
return _error_json(413, str(e), err_type="invalid_request_error")
except Exception:
logger.exception("Error parsing upload")
return _error_json(413, "File too large or invalid upload")
session_key = f"api:{session_id}" if session_id else API_SESSION_KEY
session_locks: dict[str, asyncio.Lock] = request.app["session_locks"]
session_lock = session_locks.setdefault(session_key, asyncio.Lock())
logger.info("API request session_key={} content={}", session_key, user_content[:80])
logger.info("API request session_key={} media={} text={}", session_key, len(media_paths), text[:80])
_FALLBACK = EMPTY_FINAL_RESPONSE_MESSAGE
@@ -107,7 +206,8 @@ async def handle_chat_completions(request: web.Request) -> web.Response:
try:
response = await asyncio.wait_for(
agent_loop.process_direct(
content=user_content,
content=text,
media=media_paths if media_paths else None,
session_key=session_key,
channel="api",
chat_id=API_CHAT_ID,
@@ -117,13 +217,11 @@ async def handle_chat_completions(request: web.Request) -> web.Response:
response_text = _response_text(response)
if not response_text or not response_text.strip():
logger.warning(
"Empty response for session {}, retrying",
session_key,
)
logger.warning("Empty response for session {}, retrying", session_key)
retry_response = await asyncio.wait_for(
agent_loop.process_direct(
content=user_content,
content=text,
media=media_paths if media_paths else None,
session_key=session_key,
channel="api",
chat_id=API_CHAT_ID,
@@ -132,10 +230,7 @@ async def handle_chat_completions(request: web.Request) -> web.Response:
)
response_text = _response_text(retry_response)
if not response_text or not response_text.strip():
logger.warning(
"Empty response after retry for session {}, using fallback",
session_key,
)
logger.warning("Empty response after retry, using fallback")
response_text = _FALLBACK
except asyncio.TimeoutError:
@@ -183,7 +278,7 @@ def create_app(agent_loop, model_name: str = "nanobot", request_timeout: float =
model_name: Model name reported in responses.
request_timeout: Per-request timeout in seconds.
"""
app = web.Application()
app = web.Application(client_max_size=20 * 1024 * 1024) # 20MB for base64 images
app["agent_loop"] = agent_loop
app["model_name"] = model_name
app["request_timeout"] = request_timeout
nanobot/utils/document.py
+206
View File
@@ -0,0 +1,206 @@
"""Document text extraction utilities for nanobot."""
from pathlib import Path
from loguru import logger
try:
from pypdf import PdfReader
except ImportError:
PdfReader = None # type: ignore
try:
from docx import Document as DocxDocument
except ImportError:
DocxDocument = None # type: ignore
try:
from openpyxl import load_workbook
except ImportError:
load_workbook = None # type: ignore
try:
from pptx import Presentation as PptxPresentation
except ImportError:
PptxPresentation = None # type: ignore
# Supported file extensions for text extraction
SUPPORTED_EXTENSIONS: set[str] = {
# Document formats
".pdf",
".docx",
".xlsx",
".pptx",
# Text formats
".txt",
".md",
".csv",
".json",
".xml",
".html",
".htm",
".log",
".yaml",
".yml",
".toml",
".ini",
".cfg",
# Image formats (for future OCR support)
".png",
".jpg",
".jpeg",
".gif",
".webp",
}
_MAX_TEXT_LENGTH = 200_000
def extract_text(path: Path) -> str | None:
"""Extract text from a file.
Args:
path: Path to the file.
Returns:
Extracted text as string, None for unsupported types,
or error string for failures.
"""
if not isinstance(path, Path):
path = Path(path)
if not path.exists():
return f"[error: file not found: {path}]"
ext = path.suffix.lower()
# Document formats
if ext == ".pdf":
if PdfReader is None:
return "[error: pypdf not installed]"
return _extract_pdf(path)
elif ext == ".docx":
if DocxDocument is None:
return "[error: python-docx not installed]"
return _extract_docx(path)
elif ext == ".xlsx":
if load_workbook is None:
return "[error: openpyxl not installed]"
return _extract_xlsx(path)
elif ext == ".pptx":
if PptxPresentation is None:
return "[error: python-pptx not installed]"
return _extract_pptx(path)
elif _is_text_extension(ext):
return _extract_text_file(path)
elif ext in {".png", ".jpg", ".jpeg", ".gif", ".webp"}:
# Image files - for future OCR support
return f"[image: {path.name}]"
else:
# Unsupported extension
return None
def _extract_pdf(path: Path) -> str:
"""Extract text from PDF using pypdf."""
try:
reader = PdfReader(path)
pages: list[str] = []
for i, page in enumerate(reader.pages, 1):
text = page.extract_text() or ""
pages.append(f"--- Page {i} ---\n{text}")
return _truncate("\n\n".join(pages), _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to extract PDF {}: {}", path, e)
return f"[error: failed to extract PDF: {e!s}]"
def _extract_docx(path: Path) -> str:
"""Extract text from DOCX using python-docx."""
try:
doc = DocxDocument(path)
paragraphs: list[str] = [p.text for p in doc.paragraphs if p.text.strip()]
return _truncate("\n\n".join(paragraphs), _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to extract DOCX {}: {}", path, e)
return f"[error: failed to extract DOCX: {e!s}]"
def _extract_xlsx(path: Path) -> str:
"""Extract text from XLSX using openpyxl."""
try:
wb = load_workbook(path, read_only=True, data_only=True)
sheets: list[str] = []
for sheet_name in wb.sheetnames:
ws = wb[sheet_name]
rows: list[str] = []
for row in ws.iter_rows(values_only=True):
row_text = "\t".join(str(cell) if cell is not None else "" for cell in row)
if row_text.strip():
rows.append(row_text)
if rows:
sheets.append(f"--- Sheet: {sheet_name} ---\n" + "\n".join(rows))
wb.close()
return _truncate("\n\n".join(sheets), _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to extract XLSX {}: {}", path, e)
return f"[error: failed to extract XLSX: {e!s}]"
def _extract_pptx(path: Path) -> str:
"""Extract text from PPTX using python-pptx."""
try:
prs = PptxPresentation(path)
slides: list[str] = []
for i, slide in enumerate(prs.slides, 1):
slide_text: list[str] = []
for shape in slide.shapes:
if hasattr(shape, "text") and shape.text:
slide_text.append(shape.text)
if slide_text:
slides.append(f"--- Slide {i} ---\n" + "\n".join(slide_text))
return _truncate("\n\n".join(slides), _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to extract PPTX {}: {}", path, e)
return f"[error: failed to extract PPTX: {e!s}]"
def _extract_text_file(path: Path) -> str:
"""Extract text from a plain text file."""
try:
# Try UTF-8 first, then latin-1 fallback
try:
content = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
content = path.read_text(encoding="latin-1")
return _truncate(content, _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to read text file {}: {}", path, e)
return f"[error: failed to read file: {e!s}]"
def _truncate(text: str, max_length: int) -> str:
"""Truncate text with a suffix indicating truncation."""
if len(text) <= max_length:
return text
return text[:max_length] + f"... (truncated, {len(text)} chars total)"
def _is_text_extension(ext: str) -> bool:
"""Check if extension is a text format."""
return ext in {
".txt",
".md",
".csv",
".json",
".xml",
".html",
".htm",
".log",
".yaml",
".yml",
".toml",
".ini",
".cfg",
}