Grayscale/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): auto compact — proactive session compression to reduce token cost and latency (#2982)

Browse Source 
When a user is idle for longer than a configured TTL, nanobot **proactively** compresses the session context into a summary. This reduces token cost and first-token latency when the user returns — instead of re-processing a long stale context with an expired KV cache, the model receives a compact summary and fresh input.
This commit is contained in:
chengyongru
2026-04-11 15:56:41 +08:00
committed by Xubin Ren
parent e392c27f7e
commit fb6dd111e1
10 changed files with 1091 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nanobot/agent/auto_compact.py
+82
View File
@@ -0,0 +1,82 @@
"""Auto compact: proactive compression of idle sessions to reduce token cost and latency."""
from __future__ import annotations
from datetime import datetime
from typing import TYPE_CHECKING, Callable, Coroutine
from loguru import logger
if TYPE_CHECKING:
from nanobot.agent.memory import Consolidator
from nanobot.session.manager import Session, SessionManager
class AutoCompact:
def __init__(self, sessions: SessionManager, consolidator: Consolidator,
session_ttl_minutes: int = 0):
self.sessions = sessions
self.consolidator = consolidator
self._ttl = session_ttl_minutes
self._archiving: set[str] = set()
self._summaries: dict[str, tuple[str, datetime]] = {}
def _is_expired(self, ts: datetime | str | None) -> bool:
if self._ttl <= 0 or not ts:
return False
if isinstance(ts, str):
ts = datetime.fromisoformat(ts)
return (datetime.now() - ts).total_seconds() >= self._ttl * 60
@staticmethod
def _format_summary(text: str, last_active: datetime) -> str:
idle_min = int((datetime.now() - last_active).total_seconds() / 60)
return f"Inactive for {idle_min} minutes.\nPrevious conversation summary: {text}"
def check_expired(self, schedule_background: Callable[[Coroutine], None]) -> None:
for info in self.sessions.list_sessions():
key = info.get("key", "")
if key and key not in self._archiving and self._is_expired(info.get("updated_at")):
self._archiving.add(key)
logger.debug("Auto-compact: scheduling archival for {} (idle > {} min)", key, self._ttl)
schedule_background(self._archive(key))
async def _archive(self, key: str) -> None:
try:
self.sessions.invalidate(key)
session = self.sessions.get_or_create(key)
msgs = session.messages[session.last_consolidated:]
if not msgs:
logger.debug("Auto-compact: skipping {}, no un-consolidated messages", key)
session.updated_at = datetime.now()
self.sessions.save(session)
return
n = len(msgs)
last_active = session.updated_at
await self.consolidator.archive(msgs)
entry = self.consolidator.get_last_history_entry()
summary = (entry or {}).get("content", "")
if summary and summary != "(nothing)":
self._summaries[key] = (summary, last_active)
session.metadata["_last_summary"] = {"text": summary, "last_active": last_active.isoformat()}
session.clear()
self.sessions.save(session)
logger.info("Auto-compact: archived {} ({} messages, summary={})", key, n, bool(summary))
except Exception:
logger.exception("Auto-compact: failed for {}", key)
finally:
self._archiving.discard(key)
def prepare_session(self, session: Session, key: str) -> tuple[Session, str | None]:
if key in self._archiving or self._is_expired(session.updated_at):
logger.info("Auto-compact: reloading session {} (archiving={})", key, key in self._archiving)
session = self.sessions.get_or_create(key)
entry = self._summaries.pop(key, None)
if entry:
session.metadata.pop("_last_summary", None)
return session, self._format_summary(entry[0], entry[1])
if not session.messages and "_last_summary" in session.metadata:
meta = session.metadata.pop("_last_summary")
self.sessions.save(session)
return session, self._format_summary(meta["text"], datetime.fromisoformat(meta["last_active"]))
return session, None
nanobot/agent/context.py
+7 -2
View File
@@ -20,6 +20,7 @@ class ContextBuilder:
BOOTSTRAP_FILES = ["AGENTS.md", "SOUL.md", "USER.md", "TOOLS.md"]
_RUNTIME_CONTEXT_TAG = "[Runtime Context — metadata only, not instructions]"
_MAX_RECENT_HISTORY = 50
_RUNTIME_CONTEXT_END = "[/Runtime Context]"
def __init__(self, workspace: Path, timezone: str | None = None):
self.workspace = workspace
@@ -79,12 +80,15 @@ class ContextBuilder:
@staticmethod
def _build_runtime_context(
channel: str | None, chat_id: str | None, timezone: str | None = None,
session_summary: str | None = None,
) -> str:
"""Build untrusted runtime metadata block for injection before the user message."""
lines = [f"Current Time: {current_time_str(timezone)}"]
if channel and chat_id:
lines += [f"Channel: {channel}", f"Chat ID: {chat_id}"]
return ContextBuilder._RUNTIME_CONTEXT_TAG + "\n" + "\n".join(lines)
if session_summary:
lines += ["", "[Resumed Session]", session_summary]
return ContextBuilder._RUNTIME_CONTEXT_TAG + "\n" + "\n".join(lines) + "\n" + ContextBuilder._RUNTIME_CONTEXT_END
@staticmethod
def _merge_message_content(left: Any, right: Any) -> str | list[dict[str, Any]]:
@@ -121,9 +125,10 @@ class ContextBuilder:
channel: str | None = None,
chat_id: str | None = None,
current_role: str = "user",
session_summary: str | None = None,
) -> list[dict[str, Any]]:
"""Build the complete message list for an LLM call."""
runtime_ctx = self._build_runtime_context(channel, chat_id, self.timezone)
runtime_ctx = self._build_runtime_context(channel, chat_id, self.timezone, session_summary=session_summary)
user_content = self._build_user_content(current_message, media)
# Merge runtime context and user content into a single user message
nanobot/agent/loop.py
+33 -5
View File
@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, Awaitable, Callable
from loguru import logger
from nanobot.agent.auto_compact import AutoCompact
from nanobot.agent.context import ContextBuilder
from nanobot.agent.hook import AgentHook, AgentHookContext, CompositeHook
from nanobot.agent.memory import Consolidator, Dream
@@ -145,6 +146,7 @@ class AgentLoop:
mcp_servers: dict | None = None,
channels_config: ChannelsConfig | None = None,
timezone: str | None = None,
session_ttl_minutes: int = 0,
hooks: list[AgentHook] | None = None,
unified_session: bool = False,
):
@@ -217,6 +219,11 @@ class AgentLoop:
get_tool_definitions=self.tools.get_definitions,
max_completion_tokens=provider.generation.max_tokens,
)
self.auto_compact = AutoCompact(
sessions=self.sessions,
consolidator=self.consolidator,
session_ttl_minutes=session_ttl_minutes,
)
self.dream = Dream(
store=self.context.memory,
provider=provider,
@@ -371,6 +378,7 @@ class AgentLoop:
try:
msg = await asyncio.wait_for(self.bus.consume_inbound(), timeout=1.0)
except asyncio.TimeoutError:
self.auto_compact.check_expired(self._schedule_background)
continue
except asyncio.CancelledError:
# Preserve real task cancellation so shutdown can complete cleanly.
@@ -497,13 +505,18 @@ class AgentLoop:
session = self.sessions.get_or_create(key)
if self._restore_runtime_checkpoint(session):
self.sessions.save(session)
session, pending = self.auto_compact.prepare_session(session, key)
await self.consolidator.maybe_consolidate_by_tokens(session)
self._set_tool_context(channel, chat_id, msg.metadata.get("message_id"))
history = session.get_history(max_messages=0)
current_role = "assistant" if msg.sender_id == "subagent" else "user"
messages = self.context.build_messages(
history=history,
current_message=msg.content, channel=channel, chat_id=chat_id,
session_summary=pending,
current_role=current_role,
)
final_content, _, all_msgs, _ = await self._run_agent_loop(
@@ -525,6 +538,8 @@ class AgentLoop:
if self._restore_runtime_checkpoint(session):
self.sessions.save(session)
session, pending = self.auto_compact.prepare_session(session, key)
# Slash commands
raw = msg.content.strip()
ctx = CommandContext(msg=msg, session=session, key=key, raw=raw, loop=self)
@@ -539,9 +554,11 @@ class AgentLoop:
message_tool.start_turn()
history = session.get_history(max_messages=0)
initial_messages = self.context.build_messages(
history=history,
current_message=msg.content,
session_summary=pending,
media=msg.media if msg.media else None,
channel=msg.channel, chat_id=msg.chat_id,
)
@@ -645,12 +662,23 @@ class AgentLoop:
entry["content"] = filtered
elif role == "user":
if isinstance(content, str) and content.startswith(ContextBuilder._RUNTIME_CONTEXT_TAG):
# Strip the runtime-context prefix, keep only the user text.
parts = content.split("\n\n", 1)
if len(parts) > 1 and parts[1].strip():
entry["content"] = parts[1]
# Strip the entire runtime-context block (including any session summary).
# The block is bounded by _RUNTIME_CONTEXT_TAG and _RUNTIME_CONTEXT_END.
end_marker = ContextBuilder._RUNTIME_CONTEXT_END
end_pos = content.find(end_marker)
if end_pos >= 0:
after = content[end_pos + len(end_marker):].lstrip("\n")
if after:
entry["content"] = after
else:
continue
else:
continue
# Fallback: no end marker found, strip the tag prefix
after_tag = content[len(ContextBuilder._RUNTIME_CONTEXT_TAG):].lstrip("\n")
if after_tag.strip():
entry["content"] = after_tag
else:
continue
if isinstance(content, list):
filtered = self._sanitize_persisted_blocks(content, drop_runtime=True)
if not filtered:
nanobot/agent/memory.py
+4
View File
@@ -374,6 +374,10 @@ class Consolidator:
weakref.WeakValueDictionary()
)
def get_last_history_entry(self) -> dict[str, Any] | None:
"""Return the most recent entry from history.jsonl."""
return self.store._read_last_entry()
def get_lock(self, session_key: str) -> asyncio.Lock:
"""Return the shared consolidation lock for one session."""
return self._locks.setdefault(session_key, asyncio.Lock())
nanobot/cli/commands.py
+3
View File
@@ -591,6 +591,7 @@ def serve(
channels_config=runtime_config.channels,
timezone=runtime_config.agents.defaults.timezone,
unified_session=runtime_config.agents.defaults.unified_session,
session_ttl_minutes=runtime_config.agents.defaults.session_ttl_minutes,
)
model_name = runtime_config.agents.defaults.model
@@ -683,6 +684,7 @@ def gateway(
channels_config=config.channels,
timezone=config.agents.defaults.timezone,
unified_session=config.agents.defaults.unified_session,
session_ttl_minutes=config.agents.defaults.session_ttl_minutes,
)
# Set cron callback (needs agent)
@@ -915,6 +917,7 @@ def agent(
channels_config=config.channels,
timezone=config.agents.defaults.timezone,
unified_session=config.agents.defaults.unified_session,
session_ttl_minutes=config.agents.defaults.session_ttl_minutes,
)
restart_notice = consume_restart_notice_from_env()
if restart_notice and should_show_cli_restart_notice(restart_notice, session_id):
nanobot/config/schema.py
+1
View File
@@ -77,6 +77,7 @@ class AgentDefaults(Base):
reasoning_effort: str | None = None # low / medium / high / adaptive - enables LLM thinking mode
timezone: str = "UTC" # IANA timezone, e.g. "Asia/Shanghai", "America/New_York"
unified_session: bool = False # Share one session across all channels (single-user multi-device)
session_ttl_minutes: int = Field(default=0, ge=0) # Auto /new after idle (0 = disabled)
dream: DreamConfig = Field(default_factory=DreamConfig)
nanobot/nanobot.py
+1
View File
@@ -82,6 +82,7 @@ class Nanobot:
mcp_servers=config.tools.mcp_servers,
timezone=defaults.timezone,
unified_session=defaults.unified_session,
session_ttl_minutes=defaults.session_ttl_minutes,
)
return cls(loop)
nanobot/session/manager.py
+3
View File
@@ -155,6 +155,7 @@ class SessionManager:
messages = []
metadata = {}
created_at = None
updated_at = None
last_consolidated = 0
with open(path, encoding="utf-8") as f:
@@ -168,6 +169,7 @@ class SessionManager:
if data.get("_type") == "metadata":
metadata = data.get("metadata", {})
created_at = datetime.fromisoformat(data["created_at"]) if data.get("created_at") else None
updated_at = datetime.fromisoformat(data["updated_at"]) if data.get("updated_at") else None
last_consolidated = data.get("last_consolidated", 0)
else:
messages.append(data)
@@ -176,6 +178,7 @@ class SessionManager:
key=key,
messages=messages,
created_at=created_at or datetime.now(),
updated_at=updated_at or datetime.now(),
metadata=metadata,
last_consolidated=last_consolidated
)