refactor: simplify token truncation loop
Maintainer edit: keep the strict token-budget behavior while removing the duplicate pre-loop result construction in the shared truncation helper.
This commit is contained in:
@@ -257,11 +257,11 @@ def truncate_text_to_tokens(text: str, max_tokens: int) -> str:
|
|||||||
body_budget = max_tokens - len(suffix_tokens)
|
body_budget = max_tokens - len(suffix_tokens)
|
||||||
if body_budget <= 0:
|
if body_budget <= 0:
|
||||||
return enc.decode(tokens[:max_tokens])
|
return enc.decode(tokens[:max_tokens])
|
||||||
result = enc.decode(tokens[:body_budget]) + _TRUNCATED_SUFFIX
|
for candidate_budget in range(body_budget, -1, -1):
|
||||||
while len(enc.encode(result)) > max_tokens and body_budget > 0:
|
result = enc.decode(tokens[:candidate_budget]) + _TRUNCATED_SUFFIX
|
||||||
body_budget -= 1
|
if len(enc.encode(result)) <= max_tokens:
|
||||||
result = enc.decode(tokens[:body_budget]) + _TRUNCATED_SUFFIX
|
|
||||||
return result
|
return result
|
||||||
|
return enc.decode(tokens[:max_tokens])
|
||||||
except Exception:
|
except Exception:
|
||||||
max_chars = max_tokens * 4
|
max_chars = max_tokens * 4
|
||||||
suffix_chars = len(_TRUNCATED_SUFFIX)
|
suffix_chars = len(_TRUNCATED_SUFFIX)
|
||||||
|
|||||||
Reference in New Issue
Block a user