fix: reuse token truncation helper
Maintainer edit: make token truncation include the suffix within the budget and route the consolidator through the shared helper so recent-history and archive truncation keep the same semantics.
This commit is contained in:
@@ -825,4 +825,4 @@ class TestArchiveTruncation:
|
||||
enc = tiktoken.get_encoding("cl100k_base")
|
||||
sent_content = mock_provider.chat_with_retry.call_args.kwargs["messages"][1]["content"]
|
||||
token_count = len(enc.encode(sent_content))
|
||||
assert token_count <= 9_900 + 10 # small margin for truncation suffix
|
||||
assert token_count <= 9_900
|
||||
|
||||
@@ -237,8 +237,7 @@ def test_recent_history_truncated_at_max_tokens(tmp_path) -> None:
|
||||
assert len(history_section) == 2
|
||||
|
||||
enc = tiktoken.get_encoding("cl100k_base")
|
||||
# Small margin for the truncation suffix appended after the token slice.
|
||||
assert len(enc.encode(history_section[1])) <= builder._MAX_HISTORY_TOKENS + 50
|
||||
assert len(enc.encode(history_section[1])) <= builder._MAX_HISTORY_TOKENS
|
||||
|
||||
|
||||
def test_no_recent_history_when_dream_has_processed_all(tmp_path) -> None:
|
||||
|
||||
Reference in New Issue
Block a user