fix: reuse token truncation helper

Maintainer edit: make token truncation include the suffix within the budget and route the consolidator through the shared helper so recent-history and archive truncation keep the same semantics.
This commit is contained in:
chengyongru
2026-06-17 00:47:52 +08:00
committed by Xubin Ren
parent 21d9072190
commit 072921893f
5 changed files with 21 additions and 17 deletions
+1 -2
View File
@@ -237,8 +237,7 @@ def test_recent_history_truncated_at_max_tokens(tmp_path) -> None:
assert len(history_section) == 2
enc = tiktoken.get_encoding("cl100k_base")
# Small margin for the truncation suffix appended after the token slice.
assert len(enc.encode(history_section[1])) <= builder._MAX_HISTORY_TOKENS + 50
assert len(enc.encode(history_section[1])) <= builder._MAX_HISTORY_TOKENS
def test_no_recent_history_when_dream_has_processed_all(tmp_path) -> None: