fix: stop masking runtime failures

This commit is contained in:
chengyongru
2026-07-21 11:44:52 +08:00
committed by chengyongru
parent afc65c086e
commit dfc3919b52
22 changed files with 446 additions and 300 deletions
+27 -32
View File
@@ -941,18 +941,19 @@ class Consolidator:
messages_to_summarize = public_history_messages(
summary_messages if summary_messages is not None else messages
)
formatted = MemoryStore._format_messages(messages_to_summarize)
formatted = self._truncate_to_token_budget(formatted, runtime=runtime)
system_prompt = render_template(
"agent/consolidator_archive.md",
strip=True,
)
try:
formatted = MemoryStore._format_messages(messages_to_summarize)
formatted = self._truncate_to_token_budget(formatted, runtime=runtime)
response = await runtime.provider.chat_with_retry(
model=runtime.model,
messages=[
{
"role": "system",
"content": render_template(
"agent/consolidator_archive.md",
strip=True,
),
"content": system_prompt,
},
{"role": "user", "content": formatted},
],
@@ -962,19 +963,21 @@ class Consolidator:
max_tokens=runtime.generation.max_tokens,
reasoning_effort=runtime.generation.reasoning_effort,
)
if response.finish_reason == "error":
raise RuntimeError(f"LLM returned error: {response.content}")
summary = response.content or "[no summary]"
self.store.append_history(
summary,
max_chars=_ARCHIVE_SUMMARY_MAX_CHARS,
session_key=session_key,
)
return summary
except Exception:
logger.warning("Consolidation LLM call failed, raw-dumping to history")
logger.warning("Consolidation provider call failed, raw-dumping to history")
self.store.raw_archive(messages, session_key=session_key)
return None
if response.finish_reason == "error":
logger.warning("Consolidation provider returned an error, raw-dumping to history")
self.store.raw_archive(messages, session_key=session_key)
return None
summary = response.content or "[no summary]"
self.store.append_history(
summary,
max_chars=_ARCHIVE_SUMMARY_MAX_CHARS,
session_key=session_key,
)
return summary
async def maybe_consolidate_by_tokens(
self,
@@ -1007,14 +1010,10 @@ class Consolidator:
replay_max_messages,
runtime=runtime,
)
try:
estimated, source = self.estimate_session_prompt_tokens(
session,
runtime=runtime,
)
except Exception:
logger.exception("Token estimation failed for {}", session.key)
estimated, source = 0, "error"
estimated, source = self.estimate_session_prompt_tokens(
session,
runtime=runtime,
)
if estimated <= 0:
self._persist_last_summary(session, last_summary)
return
@@ -1077,14 +1076,10 @@ class Consolidator:
# the next invocation can retry a fresh chunk.
break
try:
estimated, source = self.estimate_session_prompt_tokens(
session,
runtime=runtime,
)
except Exception:
logger.exception("Token estimation failed for {}", session.key)
estimated, source = 0, "error"
estimated, source = self.estimate_session_prompt_tokens(
session,
runtime=runtime,
)
if estimated <= 0:
break
+13 -36
View File
@@ -5,7 +5,6 @@ from __future__ import annotations
import asyncio
import inspect
import os
from contextlib import suppress
from copy import deepcopy
from dataclasses import dataclass, field
from pathlib import Path
@@ -353,37 +352,16 @@ class AgentRunner:
)
for iteration in range(spec.max_iterations):
try:
# Keep the persisted conversation untouched. Context governance
# may repair or compact historical messages for the model, but
# those synthetic edits must not shift the append boundary used
# later when the caller saves only the new turn.
messages_for_model = self.context_governor.prepare_for_model(
governance_config,
messages,
compacted_tool_call_ids,
)
except Exception:
logger.exception(
"Context governance failed on turn {} for {}; applying minimal repair",
iteration,
spec.session_key or "default",
)
try:
messages_for_model = ContextGovernor.strip_placeholder_assistant_messages(
messages
)
messages_for_model = ContextGovernor.strip_malformed_tool_calls(
messages_for_model
)
messages_for_model = ContextGovernor.drop_orphan_tool_results(
messages_for_model
)
messages_for_model = ContextGovernor.backfill_missing_tool_results(
messages_for_model
)
except Exception:
messages_for_model = messages
# Keep the persisted conversation untouched. Context governance
# may repair or compact historical messages for the model, but
# those synthetic edits must not shift the append boundary used
# later when the caller saves only the new turn. A governance
# failure must stop the run instead of sending an ungoverned copy.
messages_for_model = self.context_governor.prepare_for_model(
governance_config,
messages,
compacted_tool_call_ids,
)
context = AgentHookContext(
iteration=iteration,
messages=messages,
@@ -1167,10 +1145,9 @@ class AgentRunner:
prepare_call = getattr(spec.tools, "prepare_call", None)
tool, params, prep_error = None, tool_call.arguments, None
if callable(prepare_call):
with suppress(Exception):
prepared = prepare_call(tool_call.name, tool_call.arguments)
if isinstance(prepared, tuple) and len(prepared) == 3:
tool, params, prep_error = prepared
prepared = prepare_call(tool_call.name, tool_call.arguments)
if isinstance(prepared, tuple) and len(prepared) == 3:
tool, params, prep_error = prepared
if prep_error:
event = {
"name": tool_call.name,
+2 -19
View File
@@ -344,8 +344,6 @@ async def handle_chat_completions(request: web.Request) -> web.Response:
return resp
# -- non-streaming path (original logic) --
fallback = EMPTY_FINAL_RESPONSE_MESSAGE
try:
async with session_lock:
try:
@@ -360,24 +358,9 @@ async def handle_chat_completions(request: web.Request) -> web.Response:
timeout=timeout_s,
)
response_text = _response_text(response)
if not response_text or not response_text.strip():
logger.warning("Empty response for session {}, retrying", session_key)
retry_response = await asyncio.wait_for(
agent_loop.process_direct(
content=text,
media=media_paths if media_paths else None,
session_key=session_key,
channel="api",
chat_id=API_CHAT_ID,
persist_user_message=False,
),
timeout=timeout_s,
)
response_text = _response_text(retry_response)
if not response_text or not response_text.strip():
logger.warning("Empty response after retry, using fallback")
response_text = fallback
logger.warning("Empty response for session {}, using fallback", session_key)
response_text = EMPTY_FINAL_RESPONSE_MESSAGE
except asyncio.TimeoutError:
return _error_json(504, f"Request timed out after {timeout_s}s")
+11
View File
@@ -163,6 +163,17 @@ class AgentDefaults(Base):
) # Consolidation target ratio (0.5 = 50% of budget retained after compression)
dream: DreamConfig = Field(default_factory=DreamConfig)
@field_validator("timezone")
@classmethod
def validate_timezone(cls, value: str) -> str:
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
try:
ZoneInfo(value)
except ZoneInfoNotFoundError:
raise ValueError(f"unknown timezone {value!r}") from None
return value
class AgentsConfig(Base):
"""Agent configuration."""
+19 -30
View File
@@ -53,36 +53,32 @@ _BUNDLED_FEATURE_ALIASES = {"documents", "pdf"}
def load_pyproject(path: Path) -> dict[str, Any]:
try:
import tomllib
import tomllib
return tomllib.loads(path.read_text(encoding="utf-8"))
except Exception:
try:
content = path.read_text(encoding="utf-8")
except FileNotFoundError:
return {}
return tomllib.loads(content)
def optional_dependency_groups_from_metadata() -> dict[str, list[str] | None]:
try:
from importlib.metadata import metadata, requires
except Exception:
return {}
from importlib.metadata import metadata, requires
try:
extras = metadata("nanobot-ai").get_all("Provides-Extra") or []
groups: dict[str, list[str] | None] = {name: [] for name in extras if name != "dev"}
for raw in requires("nanobot-ai") or []:
try:
req = Requirement(raw)
except Exception:
continue
if not req.marker:
continue
for extra, deps in groups.items():
if deps is not None and req.marker.evaluate({"extra": extra}):
deps.append(raw)
return groups
except Exception:
raw_requirements = requires("nanobot-ai") or []
except PackageNotFoundError:
return {}
groups: dict[str, list[str] | None] = {name: [] for name in extras if name != "dev"}
for raw in raw_requirements:
req = Requirement(raw)
if not req.marker:
continue
for extra, deps in groups.items():
if deps is not None and req.marker.evaluate({"extra": extra}):
deps.append(raw)
return groups
def optional_dependency_groups() -> dict[str, list[str] | None]:
@@ -105,11 +101,7 @@ def optional_dependency_groups() -> dict[str, list[str] | None]:
def _install_requirements_for_extra(extra: str, deps: list[str]) -> list[str]:
install_args: list[str] = []
for raw in deps:
try:
req = Requirement(raw)
except Exception:
install_args.append(raw)
continue
req = Requirement(raw)
if req.marker and not req.marker.evaluate({"extra": extra}):
continue
req.marker = None
@@ -168,10 +160,7 @@ def _extra_dependencies_installed(
matched = False
for raw in dist.requires or []:
try:
req = Requirement(raw)
except Exception:
continue
req = Requirement(raw)
if req.marker and not req.marker.evaluate({"extra": requested_extra}):
continue
matched = True
+23 -12
View File
@@ -43,6 +43,7 @@ _TOOL_CALL_ECHO_RE = re.compile(r'^\s*(?:generate_image|message)\([^)]*\)\s*$')
_SESSION_PREVIEW_MAX_CHARS = 120
_SESSION_LIST_PREVIEW_MAX_RECORDS = 200
_SESSION_LIST_PREVIEW_MAX_CHARS = 1_000_000
_SESSION_DATA_ERRORS = (ValueError, TypeError, AttributeError, KeyError)
_FORK_VOLATILE_METADATA_KEYS = {
"goal_state",
"pending_user_turn",
@@ -466,7 +467,7 @@ class SessionManager:
if padding != 4:
stem += "=" * padding
return base64.urlsafe_b64decode(stem).decode("utf-8")
except Exception:
except _SESSION_DATA_ERRORS:
return None
def _get_session_path(self, key: str) -> Path:
@@ -491,11 +492,13 @@ class SessionManager:
if not line:
continue
data = json.loads(line)
if not isinstance(data, dict):
raise ValueError("session records must be JSON objects")
if data.get("_type") == "metadata":
stored_key = data.get("key")
return stored_key if isinstance(stored_key, str) else None
return None
except Exception:
except _SESSION_DATA_ERRORS:
return None
return None
@@ -540,11 +543,8 @@ class SessionManager:
stored_key,
)
continue
try:
shutil.move(str(fallback_path), str(path))
logger.info("Migrated session {} from {}", key, description)
except Exception:
logger.exception("Failed to migrate session {}", key)
shutil.move(str(fallback_path), str(path))
logger.info("Migrated session {} from {}", key, description)
break
if not path.exists():
@@ -564,6 +564,8 @@ class SessionManager:
continue
data = json.loads(line)
if not isinstance(data, dict):
raise ValueError("session records must be JSON objects")
if data.get("_type") == "metadata":
metadata = data.get("metadata", {})
@@ -581,7 +583,7 @@ class SessionManager:
metadata=metadata,
last_consolidated=last_consolidated
)
except Exception as e:
except _SESSION_DATA_ERRORS as e:
logger.warning("Failed to load session {}: {}", key, e)
repaired = self._repair(key)
if repaired is not None:
@@ -613,6 +615,9 @@ class SessionManager:
except json.JSONDecodeError:
skipped += 1
continue
if not isinstance(data, dict):
skipped += 1
continue
if data.get("_type") == "metadata":
metadata = data.get("metadata", {})
@@ -640,7 +645,7 @@ class SessionManager:
metadata=metadata,
last_consolidated=last_consolidated
)
except Exception as e:
except _SESSION_DATA_ERRORS as e:
logger.warning("Repair failed for session {}: {}", key, e)
return None
@@ -842,7 +847,7 @@ class SessionManager:
"metadata": metadata,
"messages": messages,
}
except Exception as e:
except _SESSION_DATA_ERRORS as e:
logger.warning("Failed to read session {}: {}", key, e)
repaired = self._repair(key)
if repaired is not None:
@@ -866,6 +871,8 @@ class SessionManager:
if not line:
continue
data = json.loads(line)
if not isinstance(data, dict):
raise ValueError("session records must be JSON objects")
if data.get("_type") != "metadata":
return None
metadata = data.get("metadata", {})
@@ -876,7 +883,7 @@ class SessionManager:
"metadata": metadata if isinstance(metadata, dict) else {},
}
return None
except Exception as e:
except _SESSION_DATA_ERRORS as e:
logger.warning("Failed to read session metadata {}: {}", key, e)
repaired = self._repair(key)
if repaired is not None:
@@ -907,6 +914,8 @@ class SessionManager:
first_line = f.readline().strip()
if first_line:
data = json.loads(first_line)
if not isinstance(data, dict):
raise ValueError("session records must be JSON objects")
if data.get("_type") == "metadata":
key = data.get("key") or fallback_key
metadata = data.get("metadata", {})
@@ -926,6 +935,8 @@ class SessionManager:
):
break
item = json.loads(line)
if not isinstance(item, dict):
raise ValueError("session records must be JSON objects")
if item.get("_type") == "metadata":
continue
text = _message_preview_text(item)
@@ -948,7 +959,7 @@ class SessionManager:
"path": str(path),
}
)
except Exception:
except _SESSION_DATA_ERRORS:
repaired = self._repair(fallback_key, path=path)
if repaired is not None:
sessions.append(
+28 -31
View File
@@ -16,6 +16,10 @@ from loguru import logger
_WORKING_TREE_DIFF_MAX_CHARS = 6000
class GitStoreError(RuntimeError):
"""Raised when the memory Git repository cannot complete an operation."""
@dataclass
class CommitInfo:
sha: str # Short SHA (8 chars)
@@ -125,9 +129,8 @@ class GitStore:
)
logger.info("Git store initialized at {}", self._workspace)
return True
except Exception:
logger.exception("Git store init failed for {}", self._workspace)
return False
except Exception as exc:
raise GitStoreError(f"Git store init failed for {self._workspace}") from exc
# -- daily operations ------------------------------------------------------
@@ -161,9 +164,8 @@ class GitStore:
sha = sha_bytes.hex()[:8]
logger.debug("Git auto-commit: {} ({})", sha, message)
return sha
except Exception:
logger.exception("Git auto-commit failed: {}", message)
return None
except Exception as exc:
raise GitStoreError(f"Git auto-commit failed: {message}") from exc
# -- internal helpers ------------------------------------------------------
@@ -190,8 +192,8 @@ class GitStore:
break
sha = commit.parents[0] if commit.parents else None
return None
except Exception:
return None
except Exception as exc:
raise GitStoreError(f"Git SHA resolution failed: {short_sha}") from exc
def _is_inside_git_repo(self) -> bool:
"""Check if self._workspace is already inside a git repository.
@@ -268,16 +270,15 @@ class GitStore:
sha = commit.parents[0] if commit.parents else None
return entries
except Exception:
logger.exception("Git log failed")
return []
except Exception as exc:
raise GitStoreError("Git log failed") from exc
def line_ages(self, file_path: str) -> list[LineAge]:
"""Compute the age of each line in a tracked file via git blame.
Returns one LineAge per line, in order.
Returns an empty list if the repo is not initialized, the file is
empty, or annotation fails.
Returns an empty list if the repo is not initialized or the file is
empty. Annotation failures raise :class:`GitStoreError`.
"""
if not self.is_initialized():
@@ -291,9 +292,8 @@ class GitStore:
from dulwich import porcelain
annotated = porcelain.annotate(str(self._workspace), file_path)
except Exception:
logger.exception("Git line_ages annotate failed for {}", file_path)
return []
except Exception as exc:
raise GitStoreError(f"Git line annotation failed for {file_path}") from exc
if not annotated:
return []
@@ -321,9 +321,8 @@ class GitStore:
outstream=out,
)
return out.getvalue().decode("utf-8", errors="replace")
except Exception:
logger.exception("Git diff_commits failed")
return ""
except Exception as exc:
raise GitStoreError(f"Git diff failed for {sha1}..{sha2}") from exc
def summarize_working_tree(self, paths: list[str]) -> str:
"""Structured summary of working-tree changes vs HEAD for *paths*.
@@ -354,8 +353,8 @@ class GitStore:
import difflib
from dulwich.repo import Repo
except ImportError:
return ""
except ImportError as exc:
raise GitStoreError("Git working-tree summary dependencies are unavailable") from exc
summary_lines: list[str] = []
diff_lines: list[str] = []
@@ -409,9 +408,8 @@ class GitStore:
total_removed += removed
summary_lines.append(f"{path}: +{added} -{removed}")
diff_lines.extend(hunks)
except Exception:
logger.exception("Git summarize_working_tree failed")
return ""
except Exception as exc:
raise GitStoreError("Git working-tree summary failed") from exc
if changed == 0:
return ""
@@ -471,9 +469,8 @@ class GitStore:
diff = self.diff_commits(parent.hex()[:8], c.sha) if parent else ""
return c, diff
return None
except Exception:
logger.exception("Git show_commit_diff failed")
return None
except Exception as exc:
raise GitStoreError(f"Git commit display failed for {short_sha}") from exc
# -- restore ---------------------------------------------------------------
@@ -485,7 +482,8 @@ class GitStore:
is provided, commits outside that history are rejected before any files
are changed.
Returns the new commit SHA, or None on failure.
Returns the new commit SHA, or ``None`` when the commit cannot be reverted.
Repository and filesystem failures raise :class:`GitStoreError`.
"""
if not self.is_initialized():
return None
@@ -534,9 +532,8 @@ class GitStore:
# Commit the restored state
msg = f"revert: undo {commit}"
return self.auto_commit(msg)
except Exception:
logger.exception("Git revert failed for {}", commit)
return None
except Exception as exc:
raise GitStoreError(f"Git revert failed for {commit}") from exc
@staticmethod
def _read_blob_from_tree(repo, tree, filepath: str) -> str | None:
+72 -49
View File
@@ -278,11 +278,7 @@ def current_time_str(timezone: str | None = None) -> str:
"""Return the current time string."""
from zoneinfo import ZoneInfo
try:
tz = ZoneInfo(timezone) if timezone else None
except (KeyError, Exception):
tz = None
tz = ZoneInfo(timezone) if timezone else None
now = datetime.now(tz=tz) if tz else datetime.now().astimezone()
offset = now.strftime("%z")
offset_fmt = f"{offset[:3]}:{offset[3:]}" if len(offset) == 5 else offset
@@ -320,8 +316,7 @@ def truncate_text_to_tokens(text: str, max_tokens: int) -> str:
Unlike :func:`truncate_text`, this measures actual tokens, so the cap holds
regardless of language or content (CJK and code cost more tokens per char).
Falls back to a char-based estimate (~4 chars/token) if tiktoken is
unavailable.
Falls back to a conservative UTF-8 byte budget if tiktoken is unavailable.
"""
if max_tokens <= 0:
return text
@@ -340,11 +335,23 @@ def truncate_text_to_tokens(text: str, max_tokens: int) -> str:
return result
return enc.decode(tokens[:max_tokens])
except Exception:
max_chars = max_tokens * 4
suffix_chars = len(_TRUNCATED_SUFFIX)
if max_chars <= suffix_chars:
return text[:max_chars]
return truncate_text(text, max_chars - suffix_chars)
if len(text.encode("utf-8")) <= max_tokens:
return text
suffix_bytes = len(_TRUNCATED_SUFFIX.encode("utf-8"))
if max_tokens <= suffix_bytes:
return _truncate_text_to_utf8_bytes(text, max_tokens)
body = _truncate_text_to_utf8_bytes(text, max_tokens - suffix_bytes)
return body + _TRUNCATED_SUFFIX
def _truncate_text_to_utf8_bytes(text: str, max_bytes: int) -> str:
"""Return the longest code-point prefix within a UTF-8 byte budget."""
if max_bytes <= 0:
return ""
encoded = text.encode("utf-8")
if len(encoded) <= max_bytes:
return text
return encoded[:max_bytes].decode("utf-8", errors="ignore")
def recent_message_start_index(
@@ -569,51 +576,67 @@ def build_assistant_message(
return msg
def estimate_prompt_tokens(
def _estimate_prompt_tokens_with_source(
messages: list[dict[str, Any]],
tools: list[dict[str, Any]] | None = None,
) -> int:
"""Estimate prompt tokens with tiktoken.
) -> tuple[int, str]:
"""Estimate prompt tokens and identify the counter used.
Counts all fields that providers send to the LLM: content, tool_calls,
reasoning_content, tool_call_id, name, plus per-message framing overhead.
"""
parts: list[str] = []
for msg in messages:
content = msg.get("content")
if isinstance(content, str):
parts.append(content)
elif isinstance(content, list):
for part in content:
if isinstance(part, dict) and part.get("type") == "text":
txt = part.get("text", "")
if txt:
parts.append(txt)
tc = msg.get("tool_calls")
if tc:
parts.append(json.dumps(tc, ensure_ascii=False))
rc = msg.get("reasoning_content")
if isinstance(rc, str) and rc:
parts.append(rc)
for key in ("name", "tool_call_id"):
value = msg.get(key)
if isinstance(value, str) and value:
parts.append(value)
message_payload = "\n".join(parts)
per_message_overhead = len(messages) * 4
try:
enc = _get_token_encoding()
parts: list[str] = []
for msg in messages:
content = msg.get("content")
if isinstance(content, str):
parts.append(content)
elif isinstance(content, list):
for part in content:
if isinstance(part, dict) and part.get("type") == "text":
txt = part.get("text", "")
if txt:
parts.append(txt)
tc = msg.get("tool_calls")
if tc:
parts.append(json.dumps(tc, ensure_ascii=False))
rc = msg.get("reasoning_content")
if isinstance(rc, str) and rc:
parts.append(rc)
for key in ("name", "tool_call_id"):
value = msg.get(key)
if isinstance(value, str) and value:
parts.append(value)
tool_tokens = (
_estimate_tools_tokens(enc, tools, leading_separator=bool(parts)) if tools else 0
)
per_message_overhead = len(messages) * 4
message_tokens = len(enc.encode("\n".join(parts))) if parts else 0
return message_tokens + tool_tokens + per_message_overhead
message_tokens = len(enc.encode(message_payload)) if message_payload else 0
return message_tokens + tool_tokens + per_message_overhead, "tiktoken"
except Exception:
return 0
tool_payload = (
("\n" if message_payload else "") + json.dumps(tools, ensure_ascii=False)
if tools
else ""
)
payload = message_payload + tool_payload
estimated = len(payload.encode("utf-8"))
return estimated + per_message_overhead, "heuristic"
def estimate_prompt_tokens(
messages: list[dict[str, Any]],
tools: list[dict[str, Any]] | None = None,
) -> int:
"""Estimate prompt tokens with tiktoken and a conservative byte fallback."""
estimated, _ = _estimate_prompt_tokens_with_source(messages, tools)
return estimated
def estimate_message_tokens(message: dict[str, Any]) -> int:
@@ -651,7 +674,7 @@ def estimate_message_tokens(message: dict[str, Any]) -> int:
enc = _get_token_encoding()
return max(4, len(enc.encode(payload)) + 4)
except Exception:
return max(4, len(payload) // 4 + 4)
return max(4, len(payload.encode("utf-8")) + 4)
def estimate_prompt_tokens_chain(
@@ -660,16 +683,16 @@ def estimate_prompt_tokens_chain(
messages: list[dict[str, Any]],
tools: list[dict[str, Any]] | None = None,
) -> tuple[int, str]:
"""Estimate prompt tokens via provider counter first, then tiktoken fallback."""
"""Estimate prompt tokens via provider, tiktoken, then a byte heuristic."""
provider_counter = getattr(provider, "estimate_prompt_tokens", None)
if callable(provider_counter):
with suppress(Exception):
tokens, source = provider_counter(messages, tools, model)
if isinstance(tokens, (int, float)) and tokens > 0:
return int(tokens), str(source or "provider_counter")
estimated = estimate_prompt_tokens(messages, tools)
estimated, source = _estimate_prompt_tokens_with_source(messages, tools)
if estimated > 0:
return int(estimated), "tiktoken"
return int(estimated), source
return 0, "none"
+1 -5
View File
@@ -758,11 +758,7 @@ def settings_payload(
config = load_config()
defaults = config.agents.defaults
active_preset_name = defaults.model_preset or "default"
try:
effective_preset = config.resolve_preset()
except Exception:
effective_preset = config.resolve_default_preset()
active_preset_name = "default"
effective_preset = config.resolve_preset()
provider_name = (
config.get_provider_name(effective_preset.model, preset=effective_preset)