w.antarandXubin Ren
973a5ee507
The recent-history section injected into the system prompt was capped by character count (_MAX_HISTORY_CHARS = 32_000). Characters are a poor proxy for tokens: ~32k chars of English is ~8k tokens, but the same char count of CJK text or code can be far more, so the cap could let the section blow well past its intended size on non English/code-heavy histories. Add a reusable truncate_text_to_tokens() helper (reusing the tiktoken cl100k_base encoder already used elsewhere, with a char-based fallback) and switch the digest cap to a token budget (_MAX_HISTORY_TOKENS = 8_000), matching the previous English-text size while holding regardless of content.
35 lines
899 B
Python
35 lines
899 B
Python
import tiktoken
|
|
|
|
from nanobot.utils.helpers import split_message, truncate_text_to_tokens
|
|
|
|
|
|
def test_split_message_no_code_blocks_unchanged():
|
|
content = "alpha beta gamma delta"
|
|
|
|
assert split_message(content, max_len=12) == ["alpha beta", "gamma delta"]
|
|
|
|
|
|
def test_truncate_text_to_tokens_keeps_text_within_budget():
|
|
text = "hello world " * 100
|
|
|
|
result = truncate_text_to_tokens(text, 10_000)
|
|
|
|
assert result == text
|
|
|
|
|
|
def test_truncate_text_to_tokens_truncates_over_budget():
|
|
enc = tiktoken.get_encoding("cl100k_base")
|
|
text = "word " * 1_000
|
|
|
|
result = truncate_text_to_tokens(text, 50)
|
|
|
|
assert result.endswith("\n... (truncated)")
|
|
body = result[: -len("\n... (truncated)")]
|
|
assert len(enc.encode(body)) <= 50
|
|
|
|
|
|
def test_truncate_text_to_tokens_non_positive_budget_returns_text():
|
|
text = "anything"
|
|
|
|
assert truncate_text_to_tokens(text, 0) == text
|