From dcb33cf9193654534ea54ee94a35e5d099b6a033 Mon Sep 17 00:00:00 2001 From: chengyongru Date: Tue, 16 Jun 2026 18:29:03 +0800 Subject: [PATCH] refactor: simplify token truncation loop Maintainer edit: keep the strict token-budget behavior while removing the duplicate pre-loop result construction in the shared truncation helper. --- nanobot/utils/helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nanobot/utils/helpers.py b/nanobot/utils/helpers.py index 01279931..c7a91821 100644 --- a/nanobot/utils/helpers.py +++ b/nanobot/utils/helpers.py @@ -257,11 +257,11 @@ def truncate_text_to_tokens(text: str, max_tokens: int) -> str: body_budget = max_tokens - len(suffix_tokens) if body_budget <= 0: return enc.decode(tokens[:max_tokens]) - result = enc.decode(tokens[:body_budget]) + _TRUNCATED_SUFFIX - while len(enc.encode(result)) > max_tokens and body_budget > 0: - body_budget -= 1 - result = enc.decode(tokens[:body_budget]) + _TRUNCATED_SUFFIX - return result + for candidate_budget in range(body_budget, -1, -1): + result = enc.decode(tokens[:candidate_budget]) + _TRUNCATED_SUFFIX + if len(enc.encode(result)) <= max_tokens: + return result + return enc.decode(tokens[:max_tokens]) except Exception: max_chars = max_tokens * 4 suffix_chars = len(_TRUNCATED_SUFFIX)