chengyongruandXubin Ren
072921893f
Maintainer edit: make token truncation include the suffix within the budget and route the consolidator through the shared helper so recent-history and archive truncation keep the same semantics.
34 lines
854 B
Python
34 lines
854 B
Python
import tiktoken
|
|
|
|
from nanobot.utils.helpers import split_message, truncate_text_to_tokens
|
|
|
|
|
|
def test_split_message_no_code_blocks_unchanged():
|
|
content = "alpha beta gamma delta"
|
|
|
|
assert split_message(content, max_len=12) == ["alpha beta", "gamma delta"]
|
|
|
|
|
|
def test_truncate_text_to_tokens_keeps_text_within_budget():
|
|
text = "hello world " * 100
|
|
|
|
result = truncate_text_to_tokens(text, 10_000)
|
|
|
|
assert result == text
|
|
|
|
|
|
def test_truncate_text_to_tokens_truncates_over_budget():
|
|
enc = tiktoken.get_encoding("cl100k_base")
|
|
text = "word " * 1_000
|
|
|
|
result = truncate_text_to_tokens(text, 50)
|
|
|
|
assert result.endswith("\n... (truncated)")
|
|
assert len(enc.encode(result)) <= 50
|
|
|
|
|
|
def test_truncate_text_to_tokens_non_positive_budget_returns_text():
|
|
text = "anything"
|
|
|
|
assert truncate_text_to_tokens(text, 0) == text
|