fix(providers): sanitize UTF-16 surrogates at provider request boundary
Symptom ------- LLM requests intermittently fail with: 'utf-8' codec can't encode characters in position N-N+1: surrogates not allowed when messages contain emoji-heavy content (e.g. HTML with mixed emoji + JSON round-trips). This blocks the affected session until the session file is quarantined. Root cause ---------- Surrogate sanitization was only applied at the CLI entry point (nanobot/cli/commands.py: _sanitize_surrogates). Requests entering the LLM provider layer through other channels (Feishu, cron, webui, tool results, memory injection) had no defensive cleaning, so any message that happened to carry unpaired UTF-16 surrogates (from an upstream JSON round-trip with ensure_ascii=True on ill-formed input, memory rehydration, or third-party content) would blow up at json.dumps -> HTTP encode time inside the provider client. Fix --- 1. Extract sanitize_surrogates() and sanitize_surrogates_deep() into nanobot/utils/helpers.py as the single source of truth. Both use utf-16-le round-tripping with errors='surrogatepass' / 'replace', so paired surrogates reconstruct back into their real code point and lone surrogates collapse to U+FFFD. 2. Make nanobot/cli/commands.py:_sanitize_surrogates a thin wrapper that re-exports the shared helper (backward compatible). 3. Add defense-in-depth at the LLM provider boundary in nanobot/providers/base.py:_sanitize_empty_content by running sanitize_surrogates_deep over each message and its content blocks right before requests are serialized to JSON. Non-goals --------- - truncate_text() is intentionally left untouched. Python str slicing cannot split a single code point into surrogate halves, so it is not the source of lone surrogates. - session/manager storage layer is untouched. Archived sessions reproduced the failure only through the request path, not through storage. Verification ------------ - New regression suite tests/providers/test_sanitize_surrogates.py covers: paired surrogate reconstruction, lone surrogate replacement, identity return on clean input (zero allocation), deep recursion on dict/list/tuple, provider _sanitize_empty_content integration, and full utf-8 encodability of the sanitized request body. - 14/14 new tests pass; full existing test module also green. - Replayed 58 archived real session messages plus adversarial lone-surrogate injection through the provider path with no encode errors after the fix. Impact ------ - No behaviour change for clean inputs (sanitize_surrogates_deep is an identity return when no surrogate is present). - Fails-safe: unpaired surrogates degrade to U+FFFD instead of aborting the entire request.
This commit is contained in:
Kris Lu
Xubin Ren
+6
-12
@@ -78,7 +78,12 @@ from nanobot.config.paths import get_workspace_path, is_default_workspace # noq
|
||||
from nanobot.config.schema import Config # noqa: E402
|
||||
from nanobot.security.network import is_loopback_host # noqa: E402
|
||||
from nanobot.utils.evaluator import evaluate_response, resolve_evaluator_prompt # noqa: E402
|
||||
from nanobot.utils.helpers import sync_workspace_templates # noqa: E402
|
||||
from nanobot.utils.helpers import ( # noqa: E402
|
||||
sanitize_surrogates as _sanitize_surrogates,
|
||||
)
|
||||
from nanobot.utils.helpers import ( # noqa: E402
|
||||
sync_workspace_templates,
|
||||
)
|
||||
from nanobot.utils.restart import ( # noqa: E402
|
||||
consume_restart_notice_from_env,
|
||||
format_restart_completed_message,
|
||||
@@ -92,17 +97,6 @@ from nanobot.webui.build import ( # noqa: E402
|
||||
from nanobot.webui.sidebar_state import read_webui_sidebar_state # noqa: E402
|
||||
|
||||
|
||||
def _sanitize_surrogates(text: str) -> str:
|
||||
"""Reconstruct surrogate pairs into real characters; replace lone surrogates.
|
||||
|
||||
On Windows, console input may produce lone surrogate code points (e.g.
|
||||
``\\ud83d\\udc08`` for U+1F408). Round-tripping through UTF-16 reconstructs
|
||||
paired surrogates into their actual characters and replaces unpaired ones
|
||||
with U+FFFD.
|
||||
"""
|
||||
return text.encode("utf-16-le", errors="surrogatepass").decode("utf-16-le", errors="replace")
|
||||
|
||||
|
||||
def _signal_name(signum: int) -> str:
|
||||
with suppress(ValueError):
|
||||
return signal.Signals(signum).name
|
||||
|
||||
@@ -15,6 +15,8 @@ from typing import Any
|
||||
import json_repair
|
||||
from loguru import logger
|
||||
|
||||
from nanobot.utils.helpers import sanitize_surrogates_deep
|
||||
|
||||
STREAM_IDLE_TIMEOUT_ENV = "NANOBOT_STREAM_IDLE_TIMEOUT_S"
|
||||
DEFAULT_STREAM_IDLE_TIMEOUT_S = 90.0
|
||||
MAX_STREAM_IDLE_TIMEOUT_S = 3600.0
|
||||
@@ -272,7 +274,15 @@ class LLMProvider(ABC):
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_empty_content(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Sanitize message content: fix empty blocks, strip internal _meta fields."""
|
||||
"""Sanitize message content: fix empty blocks, strip internal _meta fields.
|
||||
|
||||
Also strips unpaired UTF-16 surrogate code points from every string leaf
|
||||
as a defense-in-depth pass before the payload leaves the process. Lone
|
||||
surrogates (e.g. leaking from a Windows console, prompt_toolkit history,
|
||||
or a truncated JSON round-trip) otherwise cause ``UnicodeEncodeError:
|
||||
'utf-8' codec can't encode characters ... surrogates not allowed`` when
|
||||
the HTTP client serializes the request body.
|
||||
"""
|
||||
result: list[dict[str, Any]] = []
|
||||
for raw_msg in messages:
|
||||
msg = {key: value for key, value in raw_msg.items() if key != "_meta"}
|
||||
@@ -318,7 +328,10 @@ class LLMProvider(ABC):
|
||||
continue
|
||||
|
||||
result.append(msg)
|
||||
return result
|
||||
# Defense-in-depth: scrub lone UTF-16 surrogates from every string leaf.
|
||||
# This is idempotent and no-op when messages are already clean.
|
||||
sanitized = sanitize_surrogates_deep(result)
|
||||
return sanitized if isinstance(sanitized, list) else result
|
||||
|
||||
@staticmethod
|
||||
def _tool_name(tool: dict[str, Any]) -> str:
|
||||
|
||||
@@ -21,6 +21,68 @@ _TOOLS_TOKEN_CACHE_MAX_ENTRIES = 64
|
||||
_TOOLS_TOKEN_CACHE: dict[int, tuple[tuple[int, ...], dict[bool, int]]] = {}
|
||||
|
||||
|
||||
def sanitize_surrogates(text: str) -> str:
|
||||
"""Reconstruct surrogate pairs and replace unpaired surrogates.
|
||||
|
||||
Lone UTF-16 surrogate code points (``U+D800``..``U+DFFF``) cannot be
|
||||
encoded as UTF-8 and cause ``UnicodeEncodeError`` when the message is
|
||||
serialized for an HTTP request body. This helper round-trips through
|
||||
UTF-16 to reconstruct genuine surrogate pairs (produced e.g. by Windows
|
||||
console input for emoji) and substitutes lone surrogates with
|
||||
``U+FFFD``.
|
||||
|
||||
Non-string inputs are returned unchanged so this helper is safe to call
|
||||
on arbitrary message payload leaves.
|
||||
"""
|
||||
if not isinstance(text, str):
|
||||
return text
|
||||
# Fast path: no surrogate code points → return the original object so
|
||||
# callers can rely on identity to detect an actual mutation.
|
||||
for ch in text:
|
||||
cp = ord(ch)
|
||||
if 0xD800 <= cp <= 0xDFFF:
|
||||
break
|
||||
else:
|
||||
return text
|
||||
return text.encode("utf-16-le", errors="surrogatepass").decode(
|
||||
"utf-16-le", errors="replace"
|
||||
)
|
||||
|
||||
|
||||
def sanitize_surrogates_deep(value: Any) -> Any:
|
||||
"""Recursively apply :func:`sanitize_surrogates` to every string leaf.
|
||||
|
||||
Lists and dicts are rebuilt only when a nested string actually changes,
|
||||
so the common case (no surrogates present) returns the original object
|
||||
without allocations.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
cleaned = sanitize_surrogates(value)
|
||||
return cleaned
|
||||
if isinstance(value, list):
|
||||
result_list: list[Any] = []
|
||||
mutated = False
|
||||
for item in value:
|
||||
new_item = sanitize_surrogates_deep(item)
|
||||
if new_item is not item:
|
||||
mutated = True
|
||||
result_list.append(new_item)
|
||||
return result_list if mutated else value
|
||||
if isinstance(value, dict):
|
||||
result_dict: dict[Any, Any] = {}
|
||||
mutated = False
|
||||
for key, item in value.items():
|
||||
new_item = sanitize_surrogates_deep(item)
|
||||
if new_item is not item:
|
||||
mutated = True
|
||||
result_dict[key] = new_item
|
||||
return result_dict if mutated else value
|
||||
if isinstance(value, tuple):
|
||||
result_tuple = tuple(sanitize_surrogates_deep(item) for item in value)
|
||||
return result_tuple if any(a is not b for a, b in zip(result_tuple, value)) else value
|
||||
return value
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _get_token_encoding() -> Any:
|
||||
return tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Regression tests for lone UTF-16 surrogate scrubbing in provider payloads.
|
||||
|
||||
These lock down two behaviors:
|
||||
|
||||
1. ``nanobot.utils.helpers.sanitize_surrogates`` / ``sanitize_surrogates_deep``
|
||||
produce strings that can round-trip through ``str.encode('utf-8')`` even
|
||||
when the input contains unpaired surrogates.
|
||||
|
||||
2. ``LLMProvider._sanitize_empty_content`` applies the deep sanitize as a
|
||||
defense-in-depth pass so that ``UnicodeEncodeError: 'utf-8' codec can't
|
||||
encode characters ... surrogates not allowed`` cannot escape into the
|
||||
HTTP client when messages contain emoji-heavy history plus a lone
|
||||
surrogate leak (e.g. from Windows console input or a truncated JSON
|
||||
round-trip).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from nanobot.providers.base import LLMProvider
|
||||
from nanobot.utils.helpers import (
|
||||
sanitize_surrogates,
|
||||
sanitize_surrogates_deep,
|
||||
)
|
||||
|
||||
|
||||
class TestSanitizeSurrogates:
|
||||
def test_paired_surrogates_reconstructed_to_emoji(self):
|
||||
# \uD83E\uDD16 is the UTF-16 surrogate pair for 🤖 (U+1F916).
|
||||
raw = "hello \ud83e\udd16 world"
|
||||
cleaned = sanitize_surrogates(raw)
|
||||
assert cleaned == "hello 🤖 world"
|
||||
cleaned.encode("utf-8") # must not raise
|
||||
|
||||
def test_lone_surrogate_replaced_with_fffd(self):
|
||||
raw = "hello \ud83e world" # lone high surrogate
|
||||
cleaned = sanitize_surrogates(raw)
|
||||
assert "\ud83e" not in cleaned
|
||||
# replacement char U+FFFD substitutes the unpaired surrogate
|
||||
assert "\ufffd" in cleaned
|
||||
cleaned.encode("utf-8") # must not raise
|
||||
|
||||
def test_normal_text_returned_unchanged_identity(self):
|
||||
raw = "plain ascii + 中文 + 🤖"
|
||||
assert sanitize_surrogates(raw) == raw
|
||||
|
||||
def test_non_string_returned_as_is(self):
|
||||
assert sanitize_surrogates(None) is None # type: ignore[arg-type]
|
||||
assert sanitize_surrogates(42) == 42 # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestSanitizeSurrogatesDeep:
|
||||
def test_clean_input_returns_same_object(self):
|
||||
payload = {"role": "user", "content": [{"type": "text", "text": "hi 🤖"}]}
|
||||
result = sanitize_surrogates_deep(payload)
|
||||
# No allocation on clean input: same object identity.
|
||||
assert result is payload
|
||||
|
||||
def test_lone_surrogate_in_nested_content_cleaned(self):
|
||||
dirty = {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "hello \ud83e world"},
|
||||
{"type": "text", "text": "normal"},
|
||||
],
|
||||
}
|
||||
cleaned = sanitize_surrogates_deep(dirty)
|
||||
assert cleaned is not dirty # rebuilt
|
||||
first = cleaned["content"][0]["text"]
|
||||
assert "\ud83e" not in first
|
||||
# Full payload must be UTF-8 encodable.
|
||||
import json
|
||||
|
||||
json.dumps(cleaned).encode("utf-8")
|
||||
|
||||
def test_paired_surrogates_in_list_collapse_to_emoji(self):
|
||||
dirty = ["a", "b \ud83e\udd16", "c"]
|
||||
cleaned = sanitize_surrogates_deep(dirty)
|
||||
assert cleaned == ["a", "b 🤖", "c"]
|
||||
|
||||
def test_deep_recursion_on_tuple_and_dict(self):
|
||||
dirty = (
|
||||
{"k": "\ud83e"},
|
||||
["nested", "\ud83e\udd16"],
|
||||
"\ud83e clean",
|
||||
)
|
||||
cleaned = sanitize_surrogates_deep(dirty)
|
||||
# tuple preserved as tuple
|
||||
assert isinstance(cleaned, tuple)
|
||||
assert "\ud83e" not in cleaned[0]["k"]
|
||||
assert cleaned[1][1] == "🤖"
|
||||
|
||||
|
||||
class TestProviderSanitizeEmptyContent:
|
||||
"""The provider-boundary defense-in-depth pass.
|
||||
|
||||
Lone surrogates leaking into any string leaf of a request message must
|
||||
not survive ``_sanitize_empty_content``; otherwise the HTTP client will
|
||||
raise ``UnicodeEncodeError`` when serializing the request body.
|
||||
"""
|
||||
|
||||
def test_lone_surrogate_in_string_content_scrubbed(self):
|
||||
messages = [
|
||||
{"role": "user", "content": "leak \ud83e here"},
|
||||
]
|
||||
result = LLMProvider._sanitize_empty_content(messages)
|
||||
text = result[0]["content"]
|
||||
assert "\ud83e" not in text
|
||||
text.encode("utf-8") # would raise pre-fix
|
||||
|
||||
def test_lone_surrogate_in_content_block_scrubbed(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "hi \ud83e there"},
|
||||
],
|
||||
},
|
||||
]
|
||||
result = LLMProvider._sanitize_empty_content(messages)
|
||||
text = result[0]["content"][0]["text"]
|
||||
assert "\ud83e" not in text
|
||||
text.encode("utf-8")
|
||||
|
||||
def test_paired_surrogates_reconstructed_in_content(self):
|
||||
messages = [
|
||||
{"role": "user", "content": "robot \ud83e\udd16 hello"},
|
||||
]
|
||||
result = LLMProvider._sanitize_empty_content(messages)
|
||||
assert result[0]["content"] == "robot 🤖 hello"
|
||||
|
||||
def test_clean_input_semantics_unchanged(self):
|
||||
messages = [
|
||||
{"role": "user", "content": "plain 🤖 content"},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "ok"}]},
|
||||
]
|
||||
result = LLMProvider._sanitize_empty_content(messages)
|
||||
assert result[0]["content"] == "plain 🤖 content"
|
||||
assert result[1]["content"][0]["text"] == "ok"
|
||||
|
||||
def test_full_request_body_is_utf8_encodable_after_sanitize(self):
|
||||
"""End-to-end: an emoji-heavy history plus a lone surrogate must
|
||||
no longer break utf-8 encoding of the outgoing HTTP request body."""
|
||||
import json
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a 🐱 assistant."},
|
||||
{"role": "user", "content": "🤖 fixed?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "answer with a lone \ud83e half"},
|
||||
],
|
||||
},
|
||||
]
|
||||
cleaned = LLMProvider._sanitize_empty_content(messages)
|
||||
# This is the operation that previously raised UnicodeEncodeError.
|
||||
json.dumps({"model": "x", "messages": cleaned}).encode("utf-8")
|
||||
|
||||
|
||||
class TestBackwardCompatReExport:
|
||||
def test_cli_reexport_points_to_shared_helper(self):
|
||||
"""The CLI module continues to expose ``_sanitize_surrogates`` as a
|
||||
thin alias so existing imports (e.g. ``SafeFileHistory`` in the
|
||||
legacy tests) keep working."""
|
||||
from nanobot.cli.commands import _sanitize_surrogates as cli_alias
|
||||
|
||||
assert cli_alias is sanitize_surrogates
|
||||
assert cli_alias("hello \ud83e\udd16") == "hello 🤖"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user