fix: stop masking runtime failures

tests/utils/test_token_estimation.py

@@ -1,7 +1,12 @@
 import json
 
 from nanobot.utils import helpers
-from nanobot.utils.helpers import estimate_prompt_tokens, estimate_prompt_tokens_chain
+from nanobot.utils.helpers import (
+    estimate_message_tokens,
+    estimate_prompt_tokens,
+    estimate_prompt_tokens_chain,
+    truncate_text_to_tokens,
+)
 
 
 class _NoCounterProvider:
@@ -35,6 +40,57 @@ def test_estimate_prompt_tokens_chain_falls_back_when_provider_counter_fails() -
     assert source == "tiktoken"
 
 
+def test_estimate_prompt_tokens_uses_conservative_fallback_when_tiktoken_fails(
+    monkeypatch,
+) -> None:
+    monkeypatch.setattr(
+        helpers,
+        "_get_token_encoding",
+        lambda: (_ for _ in ()).throw(RuntimeError("encoding unavailable")),
+    )
+
+    content = "你" * 1_000
+    messages = [{"role": "user", "content": content}]
+    tokens = estimate_prompt_tokens(messages)
+    chain_tokens, source = estimate_prompt_tokens_chain(
+        _NoCounterProvider(),
+        "test-model",
+        messages,
+    )
+
+    actual_tokens = len(helpers.tiktoken.get_encoding("cl100k_base").encode(content)) + 4
+    assert tokens == len(content.encode("utf-8")) + 4
+    assert tokens >= actual_tokens
+    assert chain_tokens == tokens
+    assert source == "heuristic"
+
+
+def test_estimate_message_tokens_uses_utf8_byte_fallback(monkeypatch) -> None:
+    monkeypatch.setattr(
+        helpers,
+        "_get_token_encoding",
+        lambda: (_ for _ in ()).throw(RuntimeError("encoding unavailable")),
+    )
+    content = "🙂你" * 100
+
+    assert estimate_message_tokens({"role": "user", "content": content}) == (
+        len(content.encode("utf-8")) + 4
+    )
+
+
+def test_truncate_text_to_tokens_uses_utf8_byte_budget_fallback(monkeypatch) -> None:
+    monkeypatch.setattr(
+        helpers,
+        "_get_token_encoding",
+        lambda: (_ for _ in ()).throw(RuntimeError("encoding unavailable")),
+    )
+
+    result = truncate_text_to_tokens("🙂你" * 100, 40)
+
+    assert result.endswith("\n... (truncated)")
+    assert len(result.encode("utf-8")) <= 40
+
+
 def test_estimate_prompt_tokens_caches_tools_encoding(monkeypatch) -> None:
     helpers._get_token_encoding.cache_clear()
     helpers._TOOLS_TOKEN_CACHE.clear()