fix: reuse token truncation helper

Maintainer edit: make token truncation include the suffix within the budget and route the consolidator through the shared helper so recent-history and archive truncation keep the same semantics.
This commit is contained in:
chengyongru
2026-06-17 00:47:52 +08:00
committed by Xubin Ren
parent 21d9072190
commit 072921893f
5 changed files with 21 additions and 17 deletions
+1 -2
View File
@@ -24,8 +24,7 @@ def test_truncate_text_to_tokens_truncates_over_budget():
result = truncate_text_to_tokens(text, 50)
assert result.endswith("\n... (truncated)")
body = result[: -len("\n... (truncated)")]
assert len(enc.encode(body)) <= 50
assert len(enc.encode(result)) <= 50
def test_truncate_text_to_tokens_non_positive_budget_returns_text():