Merge branch 'main' into fix/skills-yaml-frontmatter

This commit is contained in:
chengyongru
2026-04-15 16:56:23 +08:00
committed by GitHub
44 changed files with 3121 additions and 343 deletions
+1 -1
View File
@@ -21,7 +21,7 @@ def _resolve_version() -> str:
return _pkg_version("nanobot-ai")
except PackageNotFoundError:
# Source checkouts often import nanobot without installed dist-info.
return _read_pyproject_version() or "0.1.5"
return _read_pyproject_version() or "0.1.5.post1"
__version__ = _resolve_version()
+16 -7
View File
@@ -3,15 +3,14 @@
import base64
import mimetypes
import platform
from importlib.resources import files as pkg_files
from pathlib import Path
from typing import Any
from nanobot.utils.helpers import current_time_str
from nanobot.agent.memory import MemoryStore
from nanobot.utils.prompt_templates import render_template
from nanobot.agent.skills import SkillsLoader
from nanobot.utils.helpers import build_assistant_message, detect_image_mime
from nanobot.utils.helpers import build_assistant_message, current_time_str, detect_image_mime
from nanobot.utils.prompt_templates import render_template
class ContextBuilder:
@@ -41,7 +40,7 @@ class ContextBuilder:
parts.append(bootstrap)
memory = self.memory.get_memory_context()
if memory:
if memory and not self._is_template_content(self.memory.read_memory(), "memory/MEMORY.md"):
parts.append(f"# Memory\n\n{memory}")
always_skills = self.skills.get_always_skills()
@@ -50,7 +49,7 @@ class ContextBuilder:
if always_content:
parts.append(f"# Active Skills\n\n{always_content}")
skills_summary = self.skills.build_skills_summary()
skills_summary = self.skills.build_skills_summary(exclude=set(always_skills))
if skills_summary:
parts.append(render_template("agent/skills_section.md", skills_summary=skills_summary))
@@ -116,6 +115,17 @@ class ContextBuilder:
return "\n\n".join(parts) if parts else ""
@staticmethod
def _is_template_content(content: str, template_path: str) -> bool:
"""Check if *content* is identical to the bundled template (user hasn't customized it)."""
try:
tpl = pkg_files("nanobot") / "templates" / template_path
if tpl.is_file():
return content.strip() == tpl.read_text(encoding="utf-8").strip()
except Exception:
pass
return False
def build_messages(
self,
history: list[dict[str, Any]],
@@ -160,7 +170,6 @@ class ContextBuilder:
if not p.is_file():
continue
raw = p.read_bytes()
# Detect real MIME type from magic bytes; fallback to filename guess
mime = detect_image_mime(raw) or mimetypes.guess_type(path)[0]
if not mime or not mime.startswith("image/"):
continue
+23 -9
View File
@@ -17,10 +17,10 @@ from nanobot.agent.autocompact import AutoCompact
from nanobot.agent.context import ContextBuilder
from nanobot.agent.hook import AgentHook, AgentHookContext, CompositeHook
from nanobot.agent.memory import Consolidator, Dream
from nanobot.agent.runner import _MAX_INJECTIONS_PER_TURN, AgentRunSpec, AgentRunner
from nanobot.agent.runner import _MAX_INJECTIONS_PER_TURN, AgentRunner, AgentRunSpec
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
from nanobot.agent.subagent import SubagentManager
from nanobot.agent.tools.cron import CronTool
from nanobot.agent.skills import BUILTIN_SKILLS_DIR
from nanobot.agent.tools.filesystem import EditFileTool, ListDirTool, ReadFileTool, WriteFileTool
from nanobot.agent.tools.message import MessageTool
from nanobot.agent.tools.notebook import NotebookEditTool
@@ -30,12 +30,14 @@ from nanobot.agent.tools.shell import ExecTool
from nanobot.agent.tools.spawn import SpawnTool
from nanobot.agent.tools.web import WebFetchTool, WebSearchTool
from nanobot.bus.events import InboundMessage, OutboundMessage
from nanobot.command import CommandContext, CommandRouter, register_builtin_commands
from nanobot.bus.queue import MessageBus
from nanobot.command import CommandContext, CommandRouter, register_builtin_commands
from nanobot.config.schema import AgentDefaults
from nanobot.providers.base import LLMProvider
from nanobot.session.manager import Session, SessionManager
from nanobot.utils.helpers import image_placeholder_text, truncate_text as truncate_text_fn
from nanobot.utils.document import extract_documents
from nanobot.utils.helpers import image_placeholder_text
from nanobot.utils.helpers import truncate_text as truncate_text_fn
from nanobot.utils.runtime import EMPTY_FINAL_RESPONSE_MESSAGE
if TYPE_CHECKING:
@@ -383,10 +385,12 @@ class AgentLoop:
pending_msg = pending_queue.get_nowait()
except asyncio.QueueEmpty:
break
user_content = self.context._build_user_content(
pending_msg.content,
pending_msg.media if pending_msg.media else None,
)
content = pending_msg.content
media = pending_msg.media if pending_msg.media else None
if media:
content, media = extract_documents(content, media)
media = media or None
user_content = self.context._build_user_content(content, media)
runtime_ctx = self.context._build_runtime_context(
pending_msg.channel,
pending_msg.chat_id,
@@ -652,6 +656,12 @@ class AgentLoop:
content=final_content or "Background task completed.",
)
# Extract document text from media at the processing boundary so all
# channels benefit without format-specific logic in ContextBuilder.
if msg.media:
new_content, image_only = extract_documents(msg.content, msg.media)
msg = dataclasses.replace(msg, content=new_content, media=image_only)
preview = msg.content[:80] + "..." if len(msg.content) > 80 else msg.content
logger.info("Processing message from {}:{}: {}", msg.channel, msg.sender_id, preview)
@@ -952,13 +962,17 @@ class AgentLoop:
session_key: str = "cli:direct",
channel: str = "cli",
chat_id: str = "direct",
media: list[str] | None = None,
on_progress: Callable[[str], Awaitable[None]] | None = None,
on_stream: Callable[[str], Awaitable[None]] | None = None,
on_stream_end: Callable[..., Awaitable[None]] | None = None,
) -> OutboundMessage | None:
"""Process a message directly and return the outbound payload."""
await self._connect_mcp()
msg = InboundMessage(channel=channel, sender_id="user", chat_id=chat_id, content=content)
msg = InboundMessage(
channel=channel, sender_id="user", chat_id=chat_id,
content=content, media=media or [],
)
return await self._process_message(
msg,
session_key=session_key,
+91 -37
View File
@@ -134,6 +134,50 @@ class AgentRunner:
continue
messages.append(injection)
async def _try_drain_injections(
self,
spec: AgentRunSpec,
messages: list[dict[str, Any]],
assistant_message: dict[str, Any] | None,
injection_cycles: int,
*,
phase: str = "after error",
iteration: int | None = None,
) -> tuple[bool, int]:
"""Drain pending injections. Returns (should_continue, updated_cycles).
If injections are found and we haven't exceeded _MAX_INJECTION_CYCLES,
append them to *messages* (and emit a checkpoint if *assistant_message*
and *iteration* are both provided) and return (True, cycles+1) so the
caller continues the iteration loop. Otherwise return (False, cycles).
"""
if injection_cycles >= _MAX_INJECTION_CYCLES:
return False, injection_cycles
injections = await self._drain_injections(spec)
if not injections:
return False, injection_cycles
injection_cycles += 1
if assistant_message is not None:
messages.append(assistant_message)
if iteration is not None:
await self._emit_checkpoint(
spec,
{
"phase": "final_response",
"iteration": iteration,
"model": spec.model,
"assistant_message": assistant_message,
"completed_tool_results": [],
"pending_tool_calls": [],
},
)
self._append_injected_messages(messages, injections)
logger.info(
"Injected {} follow-up message(s) {} ({}/{})",
len(injections), phase, injection_cycles, _MAX_INJECTION_CYCLES,
)
return True, injection_cycles
async def _drain_injections(self, spec: AgentRunSpec) -> list[dict[str, Any]]:
"""Drain pending user messages via the injection callback.
@@ -287,6 +331,13 @@ class AgentRunner:
context.error = error
context.stop_reason = stop_reason
await hook.after_iteration(context)
should_continue, injection_cycles = await self._try_drain_injections(
spec, messages, None, injection_cycles,
phase="after tool error",
)
if should_continue:
had_injections = True
continue
break
await self._emit_checkpoint(
spec,
@@ -302,16 +353,12 @@ class AgentRunner:
empty_content_retries = 0
length_recovery_count = 0
# Checkpoint 1: drain injections after tools, before next LLM call
if injection_cycles < _MAX_INJECTION_CYCLES:
injections = await self._drain_injections(spec)
if injections:
had_injections = True
injection_cycles += 1
self._append_injected_messages(messages, injections)
logger.info(
"Injected {} follow-up message(s) after tool execution ({}/{})",
len(injections), injection_cycles, _MAX_INJECTION_CYCLES,
)
_drained, injection_cycles = await self._try_drain_injections(
spec, messages, None, injection_cycles,
phase="after tool execution",
)
if _drained:
had_injections = True
await hook.after_iteration(context)
continue
@@ -379,36 +426,18 @@ class AgentRunner:
# Check for mid-turn injections BEFORE signaling stream end.
# If injections are found we keep the stream alive (resuming=True)
# so streaming channels don't prematurely finalize the card.
_injected_after_final = False
if injection_cycles < _MAX_INJECTION_CYCLES:
injections = await self._drain_injections(spec)
if injections:
had_injections = True
injection_cycles += 1
_injected_after_final = True
if assistant_message is not None:
messages.append(assistant_message)
await self._emit_checkpoint(
spec,
{
"phase": "final_response",
"iteration": iteration,
"model": spec.model,
"assistant_message": assistant_message,
"completed_tool_results": [],
"pending_tool_calls": [],
},
)
self._append_injected_messages(messages, injections)
logger.info(
"Injected {} follow-up message(s) after final response ({}/{})",
len(injections), injection_cycles, _MAX_INJECTION_CYCLES,
)
should_continue, injection_cycles = await self._try_drain_injections(
spec, messages, assistant_message, injection_cycles,
phase="after final response",
iteration=iteration,
)
if should_continue:
had_injections = True
if hook.wants_streaming():
await hook.on_stream_end(context, resuming=_injected_after_final)
await hook.on_stream_end(context, resuming=should_continue)
if _injected_after_final:
if should_continue:
await hook.after_iteration(context)
continue
@@ -421,6 +450,13 @@ class AgentRunner:
context.error = error
context.stop_reason = stop_reason
await hook.after_iteration(context)
should_continue, injection_cycles = await self._try_drain_injections(
spec, messages, None, injection_cycles,
phase="after LLM error",
)
if should_continue:
had_injections = True
continue
break
if is_blank_text(clean):
final_content = EMPTY_FINAL_RESPONSE_MESSAGE
@@ -431,6 +467,13 @@ class AgentRunner:
context.error = error
context.stop_reason = stop_reason
await hook.after_iteration(context)
should_continue, injection_cycles = await self._try_drain_injections(
spec, messages, None, injection_cycles,
phase="after empty response",
)
if should_continue:
had_injections = True
continue
break
messages.append(assistant_message or build_assistant_message(
@@ -467,6 +510,17 @@ class AgentRunner:
max_iterations=spec.max_iterations,
)
self._append_final_message(messages, final_content)
# Drain any remaining injections so they are appended to the
# conversation history instead of being re-published as
# independent inbound messages by _dispatch's finally block.
# We ignore should_continue here because the for-loop has already
# exhausted all iterations.
drained_after_max_iterations, injection_cycles = await self._try_drain_injections(
spec, messages, None, injection_cycles,
phase="after max_iterations",
)
if drained_after_max_iterations:
had_injections = True
return AgentRunResult(
final_content=final_content,
+14 -20
View File
@@ -18,10 +18,6 @@ _STRIP_SKILL_FRONTMATTER = re.compile(
)
def _escape_xml(text: str) -> str:
return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
class SkillsLoader:
"""
Loader for agent skills.
@@ -112,39 +108,37 @@ class SkillsLoader:
]
return "\n\n---\n\n".join(parts)
def build_skills_summary(self) -> str:
def build_skills_summary(self, exclude: set[str] | None = None) -> str:
"""
Build a summary of all skills (name, description, path, availability).
This is used for progressive loading - the agent can read the full
skill content using read_file when needed.
Args:
exclude: Set of skill names to omit from the summary.
Returns:
XML-formatted skills summary.
Markdown-formatted skills summary.
"""
all_skills = self.list_skills(filter_unavailable=False)
if not all_skills:
return ""
lines: list[str] = ["<skills>"]
lines: list[str] = []
for entry in all_skills:
skill_name = entry["name"]
if exclude and skill_name in exclude:
continue
meta = self._get_skill_meta(skill_name)
available = self._check_requirements(meta)
lines.extend(
[
f' <skill available="{str(available).lower()}">',
f" <name>{_escape_xml(skill_name)}</name>",
f" <description>{_escape_xml(self._get_skill_description(skill_name))}</description>",
f" <location>{entry['path']}</location>",
]
)
if not available:
desc = self._get_skill_description(skill_name)
if available:
lines.append(f"- **{skill_name}** — {desc} `{entry['path']}`")
else:
missing = self._get_missing_requirements(meta)
if missing:
lines.append(f" <requires>{_escape_xml(missing)}</requires>")
lines.append(" </skill>")
lines.append("</skills>")
suffix = f" (unavailable: {missing})" if missing else " (unavailable)"
lines.append(f"- **{skill_name}** — {desc}{suffix} `{entry['path']}`")
return "\n".join(lines)
def _get_missing_requirements(self, skill_meta: dict) -> str:
+8
View File
@@ -262,3 +262,11 @@ class SubagentManager:
def get_running_count(self) -> int:
"""Return the number of currently running subagents."""
return len(self._running_tasks)
def get_running_count_by_session(self, session_key: str) -> int:
"""Return the number of currently running subagents for a session."""
tids = self._session_tasks.get(session_key, set())
return sum(
1 for tid in tids
if tid in self._running_tasks and not self._running_tasks[tid].done()
)
+27
View File
@@ -96,10 +96,37 @@ class WebSearchTool(Tool):
self.config = config if config is not None else WebSearchConfig()
self.proxy = proxy
def _effective_provider(self) -> str:
"""Resolve the backend that execute() will actually use."""
provider = self.config.provider.strip().lower() or "brave"
if provider == "duckduckgo":
return "duckduckgo"
if provider == "brave":
api_key = self.config.api_key or os.environ.get("BRAVE_API_KEY", "")
return "brave" if api_key else "duckduckgo"
if provider == "tavily":
api_key = self.config.api_key or os.environ.get("TAVILY_API_KEY", "")
return "tavily" if api_key else "duckduckgo"
if provider == "searxng":
base_url = (self.config.base_url or os.environ.get("SEARXNG_BASE_URL", "")).strip()
return "searxng" if base_url else "duckduckgo"
if provider == "jina":
api_key = self.config.api_key or os.environ.get("JINA_API_KEY", "")
return "jina" if api_key else "duckduckgo"
if provider == "kagi":
api_key = self.config.api_key or os.environ.get("KAGI_API_KEY", "")
return "kagi" if api_key else "duckduckgo"
return provider
@property
def read_only(self) -> bool:
return True
@property
def exclusive(self) -> bool:
"""DuckDuckGo searches are serialized because ddgs is not concurrency-safe."""
return self._effective_provider() == "duckduckgo"
async def execute(self, query: str, count: int | None = None, **kwargs: Any) -> str:
provider = self.config.provider.strip().lower() or "brave"
n = min(max(count or self.config.max_results, 1), 10)
+139 -40
View File
@@ -7,15 +7,28 @@ All requests route to a single persistent API session.
from __future__ import annotations
import asyncio
import base64
import mimetypes
import re
import time
import uuid
from pathlib import Path
from typing import Any
from aiohttp import web
from loguru import logger
from nanobot.config.paths import get_media_dir
from nanobot.utils.helpers import safe_filename
from nanobot.utils.runtime import EMPTY_FINAL_RESPONSE_MESSAGE
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MB
_DATA_URL_RE = re.compile(r"^data:([^;]+);base64,(.+)$", re.DOTALL)
class _FileSizeExceeded(Exception):
"""Raised when an uploaded file exceeds the size limit."""
API_SESSION_KEY = "api:default"
API_CHAT_ID = "default"
@@ -57,48 +70,138 @@ def _response_text(value: Any) -> str:
return str(value)
# ---------------------------------------------------------------------------
# Upload helpers
# ---------------------------------------------------------------------------
def _save_base64_data_url(data_url: str, media_dir: Path) -> str | None:
"""Decode a data:...;base64,... URL and save to disk."""
m = _DATA_URL_RE.match(data_url)
if not m:
return None
mime_type, b64_payload = m.group(1), m.group(2)
try:
raw = base64.b64decode(b64_payload)
except Exception:
return None
if len(raw) > MAX_FILE_SIZE:
raise _FileSizeExceeded(
f"File exceeds {MAX_FILE_SIZE // (1024 * 1024)}MB limit"
)
ext = mimetypes.guess_extension(mime_type) or ".bin"
filename = f"{uuid.uuid4().hex[:12]}{ext}"
dest = media_dir / safe_filename(filename)
dest.write_bytes(raw)
return str(dest)
def _parse_json_content(body: dict) -> tuple[str, list[str]]:
"""Parse JSON request body. Returns (text, media_paths)."""
messages = body.get("messages")
if not isinstance(messages, list) or len(messages) != 1:
raise ValueError("Only a single user message is supported")
message = messages[0]
if not isinstance(message, dict) or message.get("role") != "user":
raise ValueError("Only a single user message is supported")
user_content = message.get("content", "")
media_dir = get_media_dir("api")
media_paths: list[str] = []
if isinstance(user_content, list):
text_parts: list[str] = []
for part in user_content:
if not isinstance(part, dict):
continue
if part.get("type") == "text":
text_parts.append(part.get("text", ""))
elif part.get("type") == "image_url":
url = part.get("image_url", {}).get("url", "")
if url.startswith("data:"):
saved = _save_base64_data_url(url, media_dir)
if saved:
media_paths.append(saved)
text = " ".join(text_parts)
elif isinstance(user_content, str):
text = user_content
else:
raise ValueError("Invalid content format")
return text, media_paths
async def _parse_multipart(request: web.Request) -> tuple[str, list[str], str | None]:
"""Parse multipart/form-data. Returns (text, media_paths, session_id)."""
media_dir = get_media_dir("api")
reader = await request.multipart()
text = ""
session_id = None
media_paths: list[str] = []
while True:
part = await reader.next()
if part is None:
break
if part.name == "message":
text = (await part.read()).decode("utf-8")
elif part.name == "session_id":
session_id = (await part.read()).decode("utf-8").strip()
elif part.name == "files":
raw = await part.read()
if len(raw) > MAX_FILE_SIZE:
raise _FileSizeExceeded(f"File '{part.filename}' exceeds {MAX_FILE_SIZE // (1024*1024)}MB limit")
filename = safe_filename(part.filename or f"{uuid.uuid4().hex[:12]}.bin")
dest = media_dir / filename
dest.write_bytes(raw)
media_paths.append(str(dest))
if not text:
text = "请分析上传的文件"
return text, media_paths, session_id
# ---------------------------------------------------------------------------
# Route handlers
# ---------------------------------------------------------------------------
async def handle_chat_completions(request: web.Request) -> web.Response:
"""POST /v1/chat/completions"""
# --- Parse body ---
try:
body = await request.json()
except Exception:
return _error_json(400, "Invalid JSON body")
messages = body.get("messages")
if not isinstance(messages, list) or len(messages) != 1:
return _error_json(400, "Only a single user message is supported")
# Stream not yet supported
if body.get("stream", False):
return _error_json(400, "stream=true is not supported yet. Set stream=false or omit it.")
message = messages[0]
if not isinstance(message, dict) or message.get("role") != "user":
return _error_json(400, "Only a single user message is supported")
user_content = message.get("content", "")
if isinstance(user_content, list):
# Multi-modal content array — extract text parts
user_content = " ".join(
part.get("text", "") for part in user_content if part.get("type") == "text"
)
"""POST /v1/chat/completions — supports JSON and multipart/form-data."""
content_type = request.content_type or ""
if not isinstance(content_type, str):
content_type = ""
agent_loop = request.app["agent_loop"]
timeout_s: float = request.app.get("request_timeout", 120.0)
model_name: str = request.app.get("model_name", "nanobot")
if (requested_model := body.get("model")) and requested_model != model_name:
return _error_json(400, f"Only configured model '{model_name}' is available")
session_key = f"api:{body['session_id']}" if body.get("session_id") else API_SESSION_KEY
try:
if content_type.startswith("multipart/"):
text, media_paths, session_id = await _parse_multipart(request)
else:
try:
body = await request.json()
except Exception:
return _error_json(400, "Invalid JSON body")
if body.get("stream", False):
return _error_json(400, "stream=true is not supported yet. Set stream=false or omit it.")
if (requested_model := body.get("model")) and requested_model != model_name:
return _error_json(400, f"Only configured model '{model_name}' is available")
text, media_paths = _parse_json_content(body)
session_id = body.get("session_id")
except ValueError as e:
return _error_json(400, str(e))
except _FileSizeExceeded as e:
return _error_json(413, str(e), err_type="invalid_request_error")
except Exception:
logger.exception("Error parsing upload")
return _error_json(413, "File too large or invalid upload")
session_key = f"api:{session_id}" if session_id else API_SESSION_KEY
session_locks: dict[str, asyncio.Lock] = request.app["session_locks"]
session_lock = session_locks.setdefault(session_key, asyncio.Lock())
logger.info("API request session_key={} content={}", session_key, user_content[:80])
logger.info("API request session_key={} media={} text={}", session_key, len(media_paths), text[:80])
_FALLBACK = EMPTY_FINAL_RESPONSE_MESSAGE
@@ -107,7 +210,8 @@ async def handle_chat_completions(request: web.Request) -> web.Response:
try:
response = await asyncio.wait_for(
agent_loop.process_direct(
content=user_content,
content=text,
media=media_paths if media_paths else None,
session_key=session_key,
channel="api",
chat_id=API_CHAT_ID,
@@ -117,13 +221,11 @@ async def handle_chat_completions(request: web.Request) -> web.Response:
response_text = _response_text(response)
if not response_text or not response_text.strip():
logger.warning(
"Empty response for session {}, retrying",
session_key,
)
logger.warning("Empty response for session {}, retrying", session_key)
retry_response = await asyncio.wait_for(
agent_loop.process_direct(
content=user_content,
content=text,
media=media_paths if media_paths else None,
session_key=session_key,
channel="api",
chat_id=API_CHAT_ID,
@@ -132,10 +234,7 @@ async def handle_chat_completions(request: web.Request) -> web.Response:
)
response_text = _response_text(retry_response)
if not response_text or not response_text.strip():
logger.warning(
"Empty response after retry for session {}, using fallback",
session_key,
)
logger.warning("Empty response after retry, using fallback")
response_text = _FALLBACK
except asyncio.TimeoutError:
@@ -183,7 +282,7 @@ def create_app(agent_loop, model_name: str = "nanobot", request_timeout: float =
model_name: Model name reported in responses.
request_timeout: Per-request timeout in seconds.
"""
app = web.Application()
app = web.Application(client_max_size=20 * 1024 * 1024) # 20MB for base64 images
app["agent_loop"] = agent_loop
app["model_name"] = model_name
app["request_timeout"] = request_timeout
+7 -1
View File
@@ -116,7 +116,13 @@ class BaseChannel(ABC):
def is_allowed(self, sender_id: str) -> bool:
"""Check if *sender_id* is permitted. Empty list → deny all; ``"*"`` → allow all."""
allow_list = getattr(self.config, "allow_from", [])
if isinstance(self.config, dict):
if "allow_from" in self.config:
allow_list = self.config.get("allow_from")
else:
allow_list = self.config.get("allowFrom", [])
else:
allow_list = getattr(self.config, "allow_from", [])
if not allow_list:
logger.warning("{}: allow_from is empty — all access denied", self.name)
return False
+44 -68
View File
@@ -1290,7 +1290,6 @@ class FeishuChannel(BaseChannel):
Supported metadata keys:
_stream_end: Finalize the streaming card.
_resuming: Mid-turn pause flush but keep the buffer alive.
_tool_hint: Delta is a formatted tool hint (for display only).
message_id: Original message id (used with _stream_end for reaction cleanup).
reaction_id: Reaction id to remove on stream end.
@@ -1309,50 +1308,44 @@ class FeishuChannel(BaseChannel):
if self.config.done_emoji and message_id:
await self._add_reaction(message_id, self.config.done_emoji)
resuming = meta.get("_resuming", False)
if resuming:
# Mid-turn pause (e.g. tool call between streaming segments).
# Flush current text to card but keep the buffer alive so the
# next segment appends to the same card.
buf = self._stream_bufs.get(chat_id)
if buf and buf.card_id and buf.text:
buf.sequence += 1
await loop.run_in_executor(
None, self._stream_update_text_sync, buf.card_id, buf.text, buf.sequence,
)
return
buf = self._stream_bufs.pop(chat_id, None)
if not buf or not buf.text:
return
# Try to finalize via streaming card; if that fails (e.g.
# streaming mode was closed by Feishu due to timeout), fall
# back to sending a regular interactive card.
if buf.card_id:
buf.sequence += 1
await loop.run_in_executor(
ok = await loop.run_in_executor(
None,
self._stream_update_text_sync,
buf.card_id,
buf.text,
buf.sequence,
)
# Required so the chat list preview exits the streaming placeholder (Feishu streaming card docs).
buf.sequence += 1
await loop.run_in_executor(
None,
self._close_streaming_mode_sync,
buf.card_id,
buf.sequence,
)
else:
for chunk in self._split_elements_by_table_limit(
self._build_card_elements(buf.text)
):
card = json.dumps(
{"config": {"wide_screen_mode": True}, "elements": chunk},
ensure_ascii=False,
)
if ok:
buf.sequence += 1
await loop.run_in_executor(
None, self._send_message_sync, rid_type, chat_id, "interactive", card
None,
self._close_streaming_mode_sync,
buf.card_id,
buf.sequence,
)
return
logger.warning(
"Streaming card {} final update failed, falling back to regular card",
buf.card_id,
)
for chunk in self._split_elements_by_table_limit(
self._build_card_elements(buf.text)
):
card = json.dumps(
{"config": {"wide_screen_mode": True}, "elements": chunk},
ensure_ascii=False,
)
await loop.run_in_executor(
None, self._send_message_sync, rid_type, chat_id, "interactive", card
)
return
# --- accumulate delta ---
@@ -1404,14 +1397,21 @@ class FeishuChannel(BaseChannel):
if buf and buf.card_id:
# Delegate to send_delta so tool hints get the same
# throttling (and card creation) as regular text deltas.
lines = self.__class__._format_tool_hint_lines(hint).split("\n")
delta = "\n\n" + "\n".join(
f"{self.config.tool_hint_prefix} {ln}" for ln in lines if ln.strip()
) + "\n\n"
await self.send_delta(msg.chat_id, delta)
await self.send_delta(
msg.chat_id,
"\n\n" + self._format_tool_hint_delta(hint) + "\n\n",
)
return
await self._send_tool_hint_card(
receive_id_type, msg.chat_id, hint
# No active streaming card — send as a regular
# interactive card with the same 🔧 prefix style.
card = json.dumps(
{"config": {"wide_screen_mode": True}, "elements": [
{"tag": "markdown", "content": self._format_tool_hint_delta(hint)},
]},
ensure_ascii=False,
)
await loop.run_in_executor(
None, self._send_message_sync, receive_id_type, msg.chat_id, "interactive", card
)
return
@@ -1708,33 +1708,9 @@ class FeishuChannel(BaseChannel):
return "\n".join(part for part in parts if part)
async def _send_tool_hint_card(
self, receive_id_type: str, receive_id: str, tool_hint: str
) -> None:
"""Send tool hint as an interactive card with formatted code block.
Args:
receive_id_type: "chat_id" or "open_id"
receive_id: The target chat or user ID
tool_hint: Formatted tool hint string (e.g., 'web_search("q"), read_file("path")')
"""
loop = asyncio.get_running_loop()
# Put each top-level tool call on its own line without altering commas inside arguments.
formatted_code = self.__class__._format_tool_hint_lines(tool_hint)
card = {
"config": {"wide_screen_mode": True},
"elements": [
{"tag": "markdown", "content": f"**Tool Calls**\n\n```text\n{formatted_code}\n```"}
],
}
await loop.run_in_executor(
None,
self._send_message_sync,
receive_id_type,
receive_id,
"interactive",
json.dumps(card, ensure_ascii=False),
def _format_tool_hint_delta(self, tool_hint: str) -> str:
"""Format a tool hint string with the 🔧 prefix for each line."""
lines = self.__class__._format_tool_hint_lines(tool_hint).split("\n")
return "\n".join(
f"{self.config.tool_hint_prefix} {ln}" for ln in lines if ln.strip()
)
+9 -1
View File
@@ -75,7 +75,15 @@ class ChannelManager:
def _validate_allow_from(self) -> None:
for name, ch in self.channels.items():
if getattr(ch.config, "allow_from", None) == []:
cfg = ch.config
if isinstance(cfg, dict):
if "allow_from" in cfg:
allow = cfg.get("allow_from")
else:
allow = cfg.get("allowFrom")
else:
allow = getattr(cfg, "allow_from", None)
if allow == []:
raise SystemExit(
f'Error: "{name}" has empty allowFrom (denies all). '
f'Set ["*"] to allow everyone, or add specific user IDs.'
+126 -6
View File
@@ -5,6 +5,7 @@ import re
from typing import Any
from loguru import logger
from pydantic import Field
from slack_sdk.socket_mode.request import SocketModeRequest
from slack_sdk.socket_mode.response import SocketModeResponse
from slack_sdk.socket_mode.websockets import SocketModeClient
@@ -13,8 +14,6 @@ from slackify_markdown import slackify_markdown
from nanobot.bus.events import OutboundMessage
from nanobot.bus.queue import MessageBus
from pydantic import Field
from nanobot.channels.base import BaseChannel
from nanobot.config.schema import Base
@@ -50,6 +49,9 @@ class SlackChannel(BaseChannel):
name = "slack"
display_name = "Slack"
_SLACK_ID_RE = re.compile(r"^[CDGUW][A-Z0-9]{2,}$")
_SLACK_CHANNEL_REF_RE = re.compile(r"^<#([A-Z0-9]+)(?:\|[^>]+)?>$")
_SLACK_USER_REF_RE = re.compile(r"^<@([A-Z0-9]+)(?:\|[^>]+)?>$")
@classmethod
def default_config(cls) -> dict[str, Any]:
@@ -63,6 +65,7 @@ class SlackChannel(BaseChannel):
self._web_client: AsyncWebClient | None = None
self._socket_client: SocketModeClient | None = None
self._bot_user_id: str | None = None
self._target_cache: dict[str, str] = {}
async def start(self) -> None:
"""Start the Slack Socket Mode client."""
@@ -113,17 +116,23 @@ class SlackChannel(BaseChannel):
logger.warning("Slack client not running")
return
try:
target_chat_id = await self._resolve_target_chat_id(msg.chat_id)
slack_meta = msg.metadata.get("slack", {}) if msg.metadata else {}
thread_ts = slack_meta.get("thread_ts")
channel_type = slack_meta.get("channel_type")
origin_chat_id = str((slack_meta.get("event", {}) or {}).get("channel") or msg.chat_id)
# Slack DMs don't use threads; channel/group replies may keep thread_ts.
thread_ts_param = thread_ts if thread_ts and channel_type != "im" else None
thread_ts_param = (
thread_ts
if thread_ts and channel_type != "im" and target_chat_id == origin_chat_id
else None
)
# Slack rejects empty text payloads. Keep media-only messages media-only,
# but send a single blank message when the bot has no text or files to send.
if msg.content or not (msg.media or []):
await self._web_client.chat_postMessage(
channel=msg.chat_id,
channel=target_chat_id,
text=self._to_mrkdwn(msg.content) if msg.content else " ",
thread_ts=thread_ts_param,
)
@@ -131,7 +140,7 @@ class SlackChannel(BaseChannel):
for media_path in msg.media or []:
try:
await self._web_client.files_upload_v2(
channel=msg.chat_id,
channel=target_chat_id,
file=media_path,
thread_ts=thread_ts_param,
)
@@ -141,12 +150,123 @@ class SlackChannel(BaseChannel):
# Update reaction emoji when the final (non-progress) response is sent
if not (msg.metadata or {}).get("_progress"):
event = slack_meta.get("event", {})
await self._update_react_emoji(msg.chat_id, event.get("ts"))
await self._update_react_emoji(origin_chat_id, event.get("ts"))
except Exception as e:
logger.error("Error sending Slack message: {}", e)
raise
async def _resolve_target_chat_id(self, target: str) -> str:
"""Resolve human-friendly Slack targets to concrete IDs when needed."""
if not self._web_client:
return target
target = target.strip()
if not target:
return target
if match := self._SLACK_CHANNEL_REF_RE.fullmatch(target):
return match.group(1)
if match := self._SLACK_USER_REF_RE.fullmatch(target):
return await self._open_dm_for_user(match.group(1))
if self._SLACK_ID_RE.fullmatch(target):
if target.startswith(("U", "W")):
return await self._open_dm_for_user(target)
return target
if target.startswith("#"):
return await self._resolve_channel_name(target[1:])
if target.startswith("@"):
return await self._resolve_user_handle(target[1:])
try:
return await self._resolve_channel_name(target)
except ValueError:
return await self._resolve_user_handle(target)
async def _resolve_channel_name(self, name: str) -> str:
normalized = self._normalize_target_name(name)
if not normalized:
raise ValueError("Slack target channel name is empty")
cache_key = f"channel:{normalized}"
if cache_key in self._target_cache:
return self._target_cache[cache_key]
cursor: str | None = None
while True:
response = await self._web_client.conversations_list(
types="public_channel,private_channel",
exclude_archived=True,
limit=200,
cursor=cursor,
)
for channel in response.get("channels", []):
if self._normalize_target_name(str(channel.get("name") or "")) == normalized:
channel_id = str(channel.get("id") or "")
if channel_id:
self._target_cache[cache_key] = channel_id
return channel_id
cursor = ((response.get("response_metadata") or {}).get("next_cursor") or "").strip()
if not cursor:
break
raise ValueError(
f"Slack channel '{name}' was not found. Use a joined channel name like "
f"'#general' or a concrete channel ID."
)
async def _resolve_user_handle(self, handle: str) -> str:
normalized = self._normalize_target_name(handle)
if not normalized:
raise ValueError("Slack target user handle is empty")
cache_key = f"user:{normalized}"
if cache_key in self._target_cache:
return self._target_cache[cache_key]
cursor: str | None = None
while True:
response = await self._web_client.users_list(limit=200, cursor=cursor)
for member in response.get("members", []):
if self._member_matches_handle(member, normalized):
user_id = str(member.get("id") or "")
if not user_id:
continue
dm_id = await self._open_dm_for_user(user_id)
self._target_cache[cache_key] = dm_id
return dm_id
cursor = ((response.get("response_metadata") or {}).get("next_cursor") or "").strip()
if not cursor:
break
raise ValueError(
f"Slack user '{handle}' was not found. Use '@name' or a concrete DM/channel ID."
)
async def _open_dm_for_user(self, user_id: str) -> str:
response = await self._web_client.conversations_open(users=user_id)
channel_id = str(((response.get("channel") or {}).get("id")) or "")
if not channel_id:
raise ValueError(f"Slack DM target for user '{user_id}' could not be opened.")
return channel_id
@staticmethod
def _normalize_target_name(value: str) -> str:
return value.strip().lstrip("#@").lower()
@classmethod
def _member_matches_handle(cls, member: dict[str, Any], normalized: str) -> bool:
profile = member.get("profile") or {}
candidates = {
str(member.get("name") or ""),
str(profile.get("display_name") or ""),
str(profile.get("display_name_normalized") or ""),
str(profile.get("real_name") or ""),
str(profile.get("real_name_normalized") or ""),
}
return normalized in {cls._normalize_target_name(candidate) for candidate in candidates if candidate}
async def _on_socket_request(
self,
client: SocketModeClient,
+11 -2
View File
@@ -302,13 +302,22 @@ class WecomChannel(BaseChannel):
elif msg_type == "mixed":
# Mixed content contains multiple message items
msg_items = body.get("mixed", {}).get("item", [])
msg_items = body.get("mixed", {}).get("msg_item", [])
for item in msg_items:
item_type = item.get("type", "")
item_type = item.get("msgtype", "")
if item_type == "text":
text = item.get("text", {}).get("content", "")
if text:
content_parts.append(text)
elif item_type == "image":
file_url = item.get("image", {}).get("url", "")
aes_key = item.get("image", {}).get("aeskey", "")
if file_url and aes_key:
file_path = await self._download_and_save_media(file_url, aes_key, "image")
if file_path:
filename = os.path.basename(file_path)
content_parts.append(f"[image: {filename}]")
media_paths.append(file_path)
else:
content_parts.append(MSG_TYPE_MAP.get(item_type, f"[{item_type}]"))
+44 -1
View File
@@ -821,6 +821,48 @@ def gateway(
console.print(f"[green]✓[/green] Heartbeat: every {hb_cfg.interval_s}s")
async def _health_server(host: str, health_port: int):
"""Lightweight HTTP health endpoint on the gateway port."""
import json as _json
async def handle(reader, writer):
try:
data = await asyncio.wait_for(reader.read(4096), timeout=5)
except (asyncio.TimeoutError, ConnectionError):
writer.close()
return
request_line = data.split(b"\r\n", 1)[0].decode("utf-8", errors="replace")
method, path = "", ""
parts = request_line.split(" ")
if len(parts) >= 2:
method, path = parts[0], parts[1]
if method == "GET" and path == "/health":
body = _json.dumps({"status": "ok"})
resp = (
f"HTTP/1.0 200 OK\r\n"
f"Content-Type: application/json\r\n"
f"Content-Length: {len(body)}\r\n"
f"\r\n{body}"
)
else:
body = "Not Found"
resp = (
f"HTTP/1.0 404 Not Found\r\n"
f"Content-Type: text/plain\r\n"
f"Content-Length: {len(body)}\r\n"
f"\r\n{body}"
)
writer.write(resp.encode())
await writer.drain()
writer.close()
server = await asyncio.start_server(handle, host, health_port)
console.print(f"[green]✓[/green] Health endpoint: http://{host}:{health_port}/health")
async with server:
await server.serve_forever()
# Register Dream system job (always-on, idempotent on restart)
dream_cfg = config.agents.defaults.dream
if dream_cfg.model_override:
@@ -843,6 +885,7 @@ def gateway(
await asyncio.gather(
agent.run(),
channels.start_all(),
_health_server(config.gateway.host, port),
)
except KeyboardInterrupt:
console.print("\nShutting down...")
@@ -964,7 +1007,7 @@ def agent(
# Interactive mode — route through bus like other channels
from nanobot.bus.events import InboundMessage
_init_prompt_session()
console.print(f"{__logo__} Interactive mode (type [bold]exit[/bold] or [bold]Ctrl+C[/bold] to quit)\n")
console.print(f"{__logo__} Interactive mode [bold blue]({config.agents.defaults.model})[/bold blue] — type [bold]exit[/bold] or [bold]Ctrl+C[/bold] to quit\n")
if ":" in session_id:
cli_channel, cli_chat_id = session_id.split(":", 1)
+1 -1
View File
@@ -102,7 +102,7 @@ class StreamRenderer:
self._live = Live(self._render(), console=c, auto_refresh=False)
self._live.start()
now = time.monotonic()
if "\n" in delta or (now - self._t) > 0.05:
if (now - self._t) > 0.15:
self._live.update(self._render())
self._live.refresh()
self._t = now
+7
View File
@@ -74,6 +74,12 @@ async def cmd_status(ctx: CommandContext) -> OutboundMessage:
search_usage_text = usage.format()
except Exception:
pass # Never let usage fetch break /status
active_tasks = loop._active_tasks.get(ctx.key, [])
task_count = sum(1 for t in active_tasks if not t.done())
try:
task_count += loop.subagents.get_running_count_by_session(ctx.key)
except Exception:
pass
return OutboundMessage(
channel=ctx.msg.channel,
chat_id=ctx.msg.chat_id,
@@ -84,6 +90,7 @@ async def cmd_status(ctx: CommandContext) -> OutboundMessage:
session_msg_count=len(session.get_history(max_messages=0)),
context_tokens_estimate=ctx_est,
search_usage_text=search_usage_text,
active_task_count=task_count,
),
metadata={**dict(ctx.msg.metadata or {}), "render_as": "text"},
)
+1 -1
View File
@@ -152,7 +152,7 @@ class ApiConfig(Base):
class GatewayConfig(Base):
"""Gateway/server configuration."""
host: str = "0.0.0.0"
host: str = "127.0.0.1" # Safer default: local-only bind.
port: int = 18790
heartbeat: HeartbeatConfig = Field(default_factory=HeartbeatConfig)
+13
View File
@@ -718,9 +718,22 @@ class LLMProvider(ABC):
identical_error_count,
(response.content or "")[:120].lower(),
)
if on_retry_wait:
await on_retry_wait(
f"Persistent retry stopped after {identical_error_count} identical errors."
)
return response
if not persistent and attempt > len(delays):
logger.warning(
"LLM request failed after {} retries, giving up: {}",
attempt,
(response.content or "")[:120].lower(),
)
if on_retry_wait:
await on_retry_wait(
f"Model request failed after {attempt} retries, giving up."
)
break
base_delay = delays[min(attempt - 1, len(delays) - 1)]
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
import json
import hashlib
import importlib.util
import os
@@ -49,6 +50,29 @@ _DEFAULT_OPENROUTER_HEADERS = {
"X-OpenRouter-Title": "nanobot",
"X-OpenRouter-Categories": "cli-agent,personal-agent",
}
_KIMI_THINKING_MODELS: frozenset[str] = frozenset({
"kimi-k2.5",
"k2.6-code-preview",
})
def _is_kimi_thinking_model(model_name: str) -> bool:
"""Return True if model_name refers to a Kimi thinking-capable model.
Supports two forms:
- Exact match: kimi-k2.5 in _KIMI_THINKING_MODELS
- Slug match: moonshotai/kimi-k2.5 -> the part after the last "/"
is checked against _KIMI_THINKING_MODELS
This covers both the native Moonshot provider (bare slug) and
OpenRouter-style names (``"publisher/slug"``).
"""
name = model_name.lower()
if name in _KIMI_THINKING_MODELS:
return True
if "/" in name and name.rsplit("/", 1)[1] in _KIMI_THINKING_MODELS:
return True
return False
def _short_tool_id() -> str:
@@ -222,6 +246,24 @@ class OpenAICompatProvider(LLMProvider):
return tool_call_id
return hashlib.sha1(tool_call_id.encode()).hexdigest()[:9]
@staticmethod
def _normalize_tool_call_arguments(arguments: Any) -> str:
"""Force function.arguments into a valid JSON object string."""
if isinstance(arguments, str):
stripped = arguments.strip()
if not stripped:
return "{}"
try:
parsed = json_repair.loads(stripped)
except Exception:
return "{}"
if isinstance(parsed, dict):
return json.dumps(parsed, ensure_ascii=False)
return "{}"
if isinstance(arguments, dict):
return json.dumps(arguments, ensure_ascii=False)
return "{}"
def _sanitize_messages(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Strip non-standard keys, normalize tool_call IDs."""
sanitized = LLMProvider._sanitize_request_messages(messages, _ALLOWED_MSG_KEYS)
@@ -241,6 +283,16 @@ class OpenAICompatProvider(LLMProvider):
continue
tc_clean = dict(tc)
tc_clean["id"] = map_id(tc_clean.get("id"))
function = tc_clean.get("function")
if isinstance(function, dict):
function_clean = dict(function)
if "arguments" in function_clean:
function_clean["arguments"] = self._normalize_tool_call_arguments(
function_clean.get("arguments")
)
else:
function_clean["arguments"] = "{}"
tc_clean["function"] = function_clean
normalized.append(tc_clean)
clean["tool_calls"] = normalized
if clean.get("role") == "assistant":
@@ -334,6 +386,16 @@ class OpenAICompatProvider(LLMProvider):
if extra:
kwargs.setdefault("extra_body", {}).update(extra)
# Model-level thinking injection for Kimi thinking-capable models.
# Strip any provider prefix (e.g. "moonshotai/") before the set lookup
# so that OpenRouter-style names like "moonshotai/kimi-k2.5" are handled
# identically to bare names like "kimi-k2.5".
if reasoning_effort is not None and _is_kimi_thinking_model(model_name):
thinking_enabled = reasoning_effort.lower() != "minimal"
kwargs.setdefault("extra_body", {}).update(
{"thinking": {"type": "enabled" if thinking_enabled else "disabled"}}
)
if tools:
kwargs["tools"] = tools
kwargs["tool_choice"] = tool_choice or "auto"
+1 -1
View File
@@ -1,6 +1,6 @@
# Skills
The following skills extend your capabilities. To use a skill, read its SKILL.md file using the read_file tool.
Skills with available="false" need dependencies installed first - you can try installing them with apt/brew.
Unavailable skills need dependencies installed first you can try installing them with apt/brew.
{{ skills_summary }}
+267
View File
@@ -0,0 +1,267 @@
"""Document text extraction utilities for nanobot."""
import mimetypes
from pathlib import Path
from loguru import logger
from nanobot.utils.helpers import detect_image_mime
try:
from pypdf import PdfReader
except ImportError:
PdfReader = None # type: ignore
try:
from docx import Document as DocxDocument
except ImportError:
DocxDocument = None # type: ignore
try:
from openpyxl import load_workbook
except ImportError:
load_workbook = None # type: ignore
try:
from pptx import Presentation as PptxPresentation
except ImportError:
PptxPresentation = None # type: ignore
# Supported file extensions for text extraction
SUPPORTED_EXTENSIONS: set[str] = {
# Document formats
".pdf",
".docx",
".xlsx",
".pptx",
# Text formats
".txt",
".md",
".csv",
".json",
".xml",
".html",
".htm",
".log",
".yaml",
".yml",
".toml",
".ini",
".cfg",
# Image formats (for future OCR support)
".png",
".jpg",
".jpeg",
".gif",
".webp",
}
_MAX_TEXT_LENGTH = 200_000
def extract_text(path: Path) -> str | None:
"""Extract text from a file.
Args:
path: Path to the file.
Returns:
Extracted text as string, None for unsupported types,
or error string for failures.
"""
if not isinstance(path, Path):
path = Path(path)
if not path.exists():
return f"[error: file not found: {path}]"
ext = path.suffix.lower()
# Document formats
if ext == ".pdf":
if PdfReader is None:
return "[error: pypdf not installed]"
return _extract_pdf(path)
elif ext == ".docx":
if DocxDocument is None:
return "[error: python-docx not installed]"
return _extract_docx(path)
elif ext == ".xlsx":
if load_workbook is None:
return "[error: openpyxl not installed]"
return _extract_xlsx(path)
elif ext == ".pptx":
if PptxPresentation is None:
return "[error: python-pptx not installed]"
return _extract_pptx(path)
elif _is_text_extension(ext):
return _extract_text_file(path)
elif ext in {".png", ".jpg", ".jpeg", ".gif", ".webp"}:
# Image files - for future OCR support
return f"[image: {path.name}]"
else:
# Unsupported extension
return None
def _extract_pdf(path: Path) -> str:
"""Extract text from PDF using pypdf."""
try:
reader = PdfReader(path)
pages: list[str] = []
for i, page in enumerate(reader.pages, 1):
text = page.extract_text() or ""
pages.append(f"--- Page {i} ---\n{text}")
return _truncate("\n\n".join(pages), _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to extract PDF {}: {}", path, e)
return f"[error: failed to extract PDF: {e!s}]"
def _extract_docx(path: Path) -> str:
"""Extract text from DOCX using python-docx."""
try:
doc = DocxDocument(path)
paragraphs: list[str] = [p.text for p in doc.paragraphs if p.text.strip()]
return _truncate("\n\n".join(paragraphs), _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to extract DOCX {}: {}", path, e)
return f"[error: failed to extract DOCX: {e!s}]"
def _extract_xlsx(path: Path) -> str:
"""Extract text from XLSX using openpyxl."""
try:
wb = load_workbook(path, read_only=True, data_only=True)
sheets: list[str] = []
for sheet_name in wb.sheetnames:
ws = wb[sheet_name]
rows: list[str] = []
for row in ws.iter_rows(values_only=True):
row_text = "\t".join(str(cell) if cell is not None else "" for cell in row)
if row_text.strip():
rows.append(row_text)
if rows:
sheets.append(f"--- Sheet: {sheet_name} ---\n" + "\n".join(rows))
wb.close()
return _truncate("\n\n".join(sheets), _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to extract XLSX {}: {}", path, e)
return f"[error: failed to extract XLSX: {e!s}]"
def _extract_pptx(path: Path) -> str:
"""Extract text from PPTX using python-pptx."""
try:
prs = PptxPresentation(path)
slides: list[str] = []
for i, slide in enumerate(prs.slides, 1):
slide_text: list[str] = []
for shape in slide.shapes:
if hasattr(shape, "text") and shape.text:
slide_text.append(shape.text)
if slide_text:
slides.append(f"--- Slide {i} ---\n" + "\n".join(slide_text))
return _truncate("\n\n".join(slides), _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to extract PPTX {}: {}", path, e)
return f"[error: failed to extract PPTX: {e!s}]"
def _extract_text_file(path: Path) -> str:
"""Extract text from a plain text file."""
try:
# Try UTF-8 first, then latin-1 fallback
try:
content = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
content = path.read_text(encoding="latin-1")
return _truncate(content, _MAX_TEXT_LENGTH)
except Exception as e:
logger.error("Failed to read text file {}: {}", path, e)
return f"[error: failed to read file: {e!s}]"
def _truncate(text: str, max_length: int) -> str:
"""Truncate text with a suffix indicating truncation."""
if len(text) <= max_length:
return text
return text[:max_length] + f"... (truncated, {len(text)} chars total)"
def _is_text_extension(ext: str) -> bool:
"""Check if extension is a text format."""
return ext in {
".txt",
".md",
".csv",
".json",
".xml",
".html",
".htm",
".log",
".yaml",
".yml",
".toml",
".ini",
".cfg",
}
# ---------------------------------------------------------------------------
# High-level helper: split media into images + extracted document text
# ---------------------------------------------------------------------------
_MAX_EXTRACT_FILE_SIZE = 50 * 1024 * 1024 # 50 MB
def extract_documents(
text: str,
media_paths: list[str],
*,
max_file_size: int = _MAX_EXTRACT_FILE_SIZE,
) -> tuple[str, list[str]]:
"""Separate images from documents in *media_paths*.
Documents (PDF, DOCX, XLSX, PPTX, plain-text, ) have their text
extracted and appended to *text*. Only image paths are kept in the
returned list so that downstream layers only need to handle vision
blocks.
Files larger than *max_file_size* bytes are skipped with a warning
to avoid unbounded memory / CPU usage.
"""
image_paths: list[str] = []
doc_texts: list[str] = []
for path_str in media_paths:
p = Path(path_str)
if not p.is_file():
continue
try:
size = p.stat().st_size
except OSError:
continue
if size > max_file_size:
logger.warning(
"Skipping oversized file for extraction: {} ({:.1f} MB > {} MB limit)",
p.name, size / (1024 * 1024), max_file_size // (1024 * 1024),
)
continue
with open(p, "rb") as f:
header = f.read(16)
mime = detect_image_mime(header) or mimetypes.guess_type(path_str)[0]
if mime and mime.startswith("image/"):
image_paths.append(path_str)
else:
extracted = extract_text(p)
if extracted and not extracted.startswith("[error:"):
doc_texts.append(f"[File: {p.name}]\n{extracted}")
if doc_texts:
text = text + "\n\n" + "\n\n".join(doc_texts)
return text, image_paths
+2
View File
@@ -400,6 +400,7 @@ def build_status_content(
session_msg_count: int,
context_tokens_estimate: int,
search_usage_text: str | None = None,
active_task_count: int = 0,
) -> str:
"""Build a human-readable runtime status snapshot.
@@ -431,6 +432,7 @@ def build_status_content(
f"\U0001f4da Context: {ctx_used_str}/{ctx_total_str} ({ctx_pct}%)",
f"\U0001f4ac Session: {session_msg_count} messages",
f"\u23f1 Uptime: {uptime}",
f"\u26a1 Tasks: {active_task_count} active",
]
if search_usage_text:
lines.append(search_usage_text)