fix(context): cap recent-history digest by tokens, not characters

The recent-history section injected into the system prompt was capped by character count (_MAX_HISTORY_CHARS = 32_000). Characters are a poor proxy for tokens: ~32k chars of English is ~8k tokens, but the same char count of CJK text or code can be far more, so the cap could let the section blow well past its intended size on non English/code-heavy histories.

Add a reusable truncate_text_to_tokens() helper (reusing the tiktoken
 cl100k_base encoder already used elsewhere, with a char-based fallback) and
  switch the digest cap to a token budget (_MAX_HISTORY_TOKENS = 8_000),
  matching the previous English-text size while holding regardless of
  content.

nanobot/agent/context.py

@@ -17,7 +17,7 @@ from nanobot.utils.helpers import (
     current_time_str,
     detect_image_mime,
     load_bundled_template,
-    truncate_text,
+    truncate_text_to_tokens,
 )
 from nanobot.utils.prompt_templates import render_template
 
@@ -54,7 +54,7 @@ class ContextBuilder:
     BOOTSTRAP_FILES = ["AGENTS.md", "SOUL.md", "USER.md"]
     _RUNTIME_CONTEXT_TAG = "[Runtime Context — metadata only, not instructions]"
     _MAX_RECENT_HISTORY = 50
-    _MAX_HISTORY_CHARS = 32_000  # hard cap on recent history section size
+    _MAX_HISTORY_TOKENS = 8_000  # hard cap on recent history section size (tokens)
     _RUNTIME_CONTEXT_END = "[/Runtime Context]"
 
     def __init__(self, workspace: Path, timezone: str | None = None, disabled_skills: list[str] | None = None):
@@ -108,7 +108,7 @@ class ContextBuilder:
                 history_text = "\n".join(
                     f"- [{e['timestamp']}] {e['content']}" for e in capped
                 )
-                history_text = truncate_text(history_text, self._MAX_HISTORY_CHARS)
+                history_text = truncate_text_to_tokens(history_text, self._MAX_HISTORY_TOKENS)
                 parts.append("# Recent History\n\n" + history_text)
 
         if session_summary:

nanobot/utils/helpers.py

@@ -237,6 +237,26 @@ def truncate_text(text: str, max_chars: int) -> str:
     return text[:max_chars] + "\n... (truncated)"
 
 
+def truncate_text_to_tokens(text: str, max_tokens: int) -> str:
+    """Truncate text to a token budget with a stable suffix.
+
+    Unlike :func:`truncate_text`, this measures actual tokens, so the cap holds
+    regardless of language or content (CJK and code cost more tokens per char).
+    Falls back to a char-based estimate (~4 chars/token) if tiktoken is
+    unavailable.
+    """
+    if max_tokens <= 0:
+        return text
+    try:
+        enc = tiktoken.get_encoding("cl100k_base")
+        tokens = enc.encode(text)
+        if len(tokens) <= max_tokens:
+            return text
+        return enc.decode(tokens[:max_tokens]) + "\n... (truncated)"
+    except Exception:
+        return truncate_text(text, max_tokens * 4)
+
+
 def find_legal_message_start(messages: list[dict[str, Any]]) -> int:
     """Find the first index whose tool results have matching assistant calls."""
     declared: set[str] = set()

tests/utils/test_helpers.py

@@ -1,7 +1,34 @@
-from nanobot.utils.helpers import split_message
+import tiktoken
+
+from nanobot.utils.helpers import split_message, truncate_text_to_tokens
 
 
 def test_split_message_no_code_blocks_unchanged():
     content = "alpha beta gamma delta"
 
     assert split_message(content, max_len=12) == ["alpha beta", "gamma delta"]
+
+
+def test_truncate_text_to_tokens_keeps_text_within_budget():
+    text = "hello world " * 100
+
+    result = truncate_text_to_tokens(text, 10_000)
+
+    assert result == text
+
+
+def test_truncate_text_to_tokens_truncates_over_budget():
+    enc = tiktoken.get_encoding("cl100k_base")
+    text = "word " * 1_000
+
+    result = truncate_text_to_tokens(text, 50)
+
+    assert result.endswith("\n... (truncated)")
+    body = result[: -len("\n... (truncated)")]
+    assert len(enc.encode(body)) <= 50
+
+
+def test_truncate_text_to_tokens_non_positive_budget_returns_text():
+    text = "anything"
+
+    assert truncate_text_to_tokens(text, 0) == text