fix: reuse token truncation helper
Maintainer edit: make token truncation include the suffix within the budget and route the consolidator through the shared helper so recent-history and archive truncation keep the same semantics.
This commit is contained in:
@@ -237,8 +237,7 @@ def test_recent_history_truncated_at_max_tokens(tmp_path) -> None:
|
||||
assert len(history_section) == 2
|
||||
|
||||
enc = tiktoken.get_encoding("cl100k_base")
|
||||
# Small margin for the truncation suffix appended after the token slice.
|
||||
assert len(enc.encode(history_section[1])) <= builder._MAX_HISTORY_TOKENS + 50
|
||||
assert len(enc.encode(history_section[1])) <= builder._MAX_HISTORY_TOKENS
|
||||
|
||||
|
||||
def test_no_recent_history_when_dream_has_processed_all(tmp_path) -> None:
|
||||
|
||||
Reference in New Issue
Block a user