From 89d8c055a82768bb7cd5e7a573e1cb4947d2a0ab Mon Sep 17 00:00:00 2001 From: Kris Lu Date: Thu, 16 Jul 2026 16:09:56 +0800 Subject: [PATCH] fix(providers): sanitize UTF-16 surrogates at provider request boundary Symptom ------- LLM requests intermittently fail with: 'utf-8' codec can't encode characters in position N-N+1: surrogates not allowed when messages contain emoji-heavy content (e.g. HTML with mixed emoji + JSON round-trips). This blocks the affected session until the session file is quarantined. Root cause ---------- Surrogate sanitization was only applied at the CLI entry point (nanobot/cli/commands.py: _sanitize_surrogates). Requests entering the LLM provider layer through other channels (Feishu, cron, webui, tool results, memory injection) had no defensive cleaning, so any message that happened to carry unpaired UTF-16 surrogates (from an upstream JSON round-trip with ensure_ascii=True on ill-formed input, memory rehydration, or third-party content) would blow up at json.dumps -> HTTP encode time inside the provider client. Fix --- 1. Extract sanitize_surrogates() and sanitize_surrogates_deep() into nanobot/utils/helpers.py as the single source of truth. Both use utf-16-le round-tripping with errors='surrogatepass' / 'replace', so paired surrogates reconstruct back into their real code point and lone surrogates collapse to U+FFFD. 2. Make nanobot/cli/commands.py:_sanitize_surrogates a thin wrapper that re-exports the shared helper (backward compatible). 3. Add defense-in-depth at the LLM provider boundary in nanobot/providers/base.py:_sanitize_empty_content by running sanitize_surrogates_deep over each message and its content blocks right before requests are serialized to JSON. Non-goals --------- - truncate_text() is intentionally left untouched. Python str slicing cannot split a single code point into surrogate halves, so it is not the source of lone surrogates. - session/manager storage layer is untouched. Archived sessions reproduced the failure only through the request path, not through storage. Verification ------------ - New regression suite tests/providers/test_sanitize_surrogates.py covers: paired surrogate reconstruction, lone surrogate replacement, identity return on clean input (zero allocation), deep recursion on dict/list/tuple, provider _sanitize_empty_content integration, and full utf-8 encodability of the sanitized request body. - 14/14 new tests pass; full existing test module also green. - Replayed 58 archived real session messages plus adversarial lone-surrogate injection through the provider path with no encode errors after the fix. Impact ------ - No behaviour change for clean inputs (sanitize_surrogates_deep is an identity return when no surrogate is present). - Fails-safe: unpaired surrogates degrade to U+FFFD instead of aborting the entire request. --- nanobot/cli/commands.py | 18 +- nanobot/providers/base.py | 17 +- nanobot/utils/helpers.py | 62 +++++++ tests/providers/test_sanitize_surrogates.py | 174 ++++++++++++++++++++ 4 files changed, 257 insertions(+), 14 deletions(-) create mode 100644 tests/providers/test_sanitize_surrogates.py diff --git a/nanobot/cli/commands.py b/nanobot/cli/commands.py index 6cdf49c4..5965357e 100644 --- a/nanobot/cli/commands.py +++ b/nanobot/cli/commands.py @@ -78,7 +78,12 @@ from nanobot.config.paths import get_workspace_path, is_default_workspace # noq from nanobot.config.schema import Config # noqa: E402 from nanobot.security.network import is_loopback_host # noqa: E402 from nanobot.utils.evaluator import evaluate_response, resolve_evaluator_prompt # noqa: E402 -from nanobot.utils.helpers import sync_workspace_templates # noqa: E402 +from nanobot.utils.helpers import ( # noqa: E402 + sanitize_surrogates as _sanitize_surrogates, +) +from nanobot.utils.helpers import ( # noqa: E402 + sync_workspace_templates, +) from nanobot.utils.restart import ( # noqa: E402 consume_restart_notice_from_env, format_restart_completed_message, @@ -92,17 +97,6 @@ from nanobot.webui.build import ( # noqa: E402 from nanobot.webui.sidebar_state import read_webui_sidebar_state # noqa: E402 -def _sanitize_surrogates(text: str) -> str: - """Reconstruct surrogate pairs into real characters; replace lone surrogates. - - On Windows, console input may produce lone surrogate code points (e.g. - ``\\ud83d\\udc08`` for U+1F408). Round-tripping through UTF-16 reconstructs - paired surrogates into their actual characters and replaces unpaired ones - with U+FFFD. - """ - return text.encode("utf-16-le", errors="surrogatepass").decode("utf-16-le", errors="replace") - - def _signal_name(signum: int) -> str: with suppress(ValueError): return signal.Signals(signum).name diff --git a/nanobot/providers/base.py b/nanobot/providers/base.py index 1a1be5ed..84610878 100644 --- a/nanobot/providers/base.py +++ b/nanobot/providers/base.py @@ -15,6 +15,8 @@ from typing import Any import json_repair from loguru import logger +from nanobot.utils.helpers import sanitize_surrogates_deep + STREAM_IDLE_TIMEOUT_ENV = "NANOBOT_STREAM_IDLE_TIMEOUT_S" DEFAULT_STREAM_IDLE_TIMEOUT_S = 90.0 MAX_STREAM_IDLE_TIMEOUT_S = 3600.0 @@ -272,7 +274,15 @@ class LLMProvider(ABC): @staticmethod def _sanitize_empty_content(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Sanitize message content: fix empty blocks, strip internal _meta fields.""" + """Sanitize message content: fix empty blocks, strip internal _meta fields. + + Also strips unpaired UTF-16 surrogate code points from every string leaf + as a defense-in-depth pass before the payload leaves the process. Lone + surrogates (e.g. leaking from a Windows console, prompt_toolkit history, + or a truncated JSON round-trip) otherwise cause ``UnicodeEncodeError: + 'utf-8' codec can't encode characters ... surrogates not allowed`` when + the HTTP client serializes the request body. + """ result: list[dict[str, Any]] = [] for raw_msg in messages: msg = {key: value for key, value in raw_msg.items() if key != "_meta"} @@ -318,7 +328,10 @@ class LLMProvider(ABC): continue result.append(msg) - return result + # Defense-in-depth: scrub lone UTF-16 surrogates from every string leaf. + # This is idempotent and no-op when messages are already clean. + sanitized = sanitize_surrogates_deep(result) + return sanitized if isinstance(sanitized, list) else result @staticmethod def _tool_name(tool: dict[str, Any]) -> str: diff --git a/nanobot/utils/helpers.py b/nanobot/utils/helpers.py index 3925e785..64e27770 100644 --- a/nanobot/utils/helpers.py +++ b/nanobot/utils/helpers.py @@ -21,6 +21,68 @@ _TOOLS_TOKEN_CACHE_MAX_ENTRIES = 64 _TOOLS_TOKEN_CACHE: dict[int, tuple[tuple[int, ...], dict[bool, int]]] = {} +def sanitize_surrogates(text: str) -> str: + """Reconstruct surrogate pairs and replace unpaired surrogates. + + Lone UTF-16 surrogate code points (``U+D800``..``U+DFFF``) cannot be + encoded as UTF-8 and cause ``UnicodeEncodeError`` when the message is + serialized for an HTTP request body. This helper round-trips through + UTF-16 to reconstruct genuine surrogate pairs (produced e.g. by Windows + console input for emoji) and substitutes lone surrogates with + ``U+FFFD``. + + Non-string inputs are returned unchanged so this helper is safe to call + on arbitrary message payload leaves. + """ + if not isinstance(text, str): + return text + # Fast path: no surrogate code points β†’ return the original object so + # callers can rely on identity to detect an actual mutation. + for ch in text: + cp = ord(ch) + if 0xD800 <= cp <= 0xDFFF: + break + else: + return text + return text.encode("utf-16-le", errors="surrogatepass").decode( + "utf-16-le", errors="replace" + ) + + +def sanitize_surrogates_deep(value: Any) -> Any: + """Recursively apply :func:`sanitize_surrogates` to every string leaf. + + Lists and dicts are rebuilt only when a nested string actually changes, + so the common case (no surrogates present) returns the original object + without allocations. + """ + if isinstance(value, str): + cleaned = sanitize_surrogates(value) + return cleaned + if isinstance(value, list): + result_list: list[Any] = [] + mutated = False + for item in value: + new_item = sanitize_surrogates_deep(item) + if new_item is not item: + mutated = True + result_list.append(new_item) + return result_list if mutated else value + if isinstance(value, dict): + result_dict: dict[Any, Any] = {} + mutated = False + for key, item in value.items(): + new_item = sanitize_surrogates_deep(item) + if new_item is not item: + mutated = True + result_dict[key] = new_item + return result_dict if mutated else value + if isinstance(value, tuple): + result_tuple = tuple(sanitize_surrogates_deep(item) for item in value) + return result_tuple if any(a is not b for a, b in zip(result_tuple, value)) else value + return value + + @lru_cache(maxsize=1) def _get_token_encoding() -> Any: return tiktoken.get_encoding("cl100k_base") diff --git a/tests/providers/test_sanitize_surrogates.py b/tests/providers/test_sanitize_surrogates.py new file mode 100644 index 00000000..9d69eff0 --- /dev/null +++ b/tests/providers/test_sanitize_surrogates.py @@ -0,0 +1,174 @@ +"""Regression tests for lone UTF-16 surrogate scrubbing in provider payloads. + +These lock down two behaviors: + +1. ``nanobot.utils.helpers.sanitize_surrogates`` / ``sanitize_surrogates_deep`` + produce strings that can round-trip through ``str.encode('utf-8')`` even + when the input contains unpaired surrogates. + +2. ``LLMProvider._sanitize_empty_content`` applies the deep sanitize as a + defense-in-depth pass so that ``UnicodeEncodeError: 'utf-8' codec can't + encode characters ... surrogates not allowed`` cannot escape into the + HTTP client when messages contain emoji-heavy history plus a lone + surrogate leak (e.g. from Windows console input or a truncated JSON + round-trip). +""" + +from __future__ import annotations + +import pytest + +from nanobot.providers.base import LLMProvider +from nanobot.utils.helpers import ( + sanitize_surrogates, + sanitize_surrogates_deep, +) + + +class TestSanitizeSurrogates: + def test_paired_surrogates_reconstructed_to_emoji(self): + # \uD83E\uDD16 is the UTF-16 surrogate pair for πŸ€– (U+1F916). + raw = "hello \ud83e\udd16 world" + cleaned = sanitize_surrogates(raw) + assert cleaned == "hello πŸ€– world" + cleaned.encode("utf-8") # must not raise + + def test_lone_surrogate_replaced_with_fffd(self): + raw = "hello \ud83e world" # lone high surrogate + cleaned = sanitize_surrogates(raw) + assert "\ud83e" not in cleaned + # replacement char U+FFFD substitutes the unpaired surrogate + assert "\ufffd" in cleaned + cleaned.encode("utf-8") # must not raise + + def test_normal_text_returned_unchanged_identity(self): + raw = "plain ascii + δΈ­ζ–‡ + πŸ€–" + assert sanitize_surrogates(raw) == raw + + def test_non_string_returned_as_is(self): + assert sanitize_surrogates(None) is None # type: ignore[arg-type] + assert sanitize_surrogates(42) == 42 # type: ignore[arg-type] + + +class TestSanitizeSurrogatesDeep: + def test_clean_input_returns_same_object(self): + payload = {"role": "user", "content": [{"type": "text", "text": "hi πŸ€–"}]} + result = sanitize_surrogates_deep(payload) + # No allocation on clean input: same object identity. + assert result is payload + + def test_lone_surrogate_in_nested_content_cleaned(self): + dirty = { + "role": "user", + "content": [ + {"type": "text", "text": "hello \ud83e world"}, + {"type": "text", "text": "normal"}, + ], + } + cleaned = sanitize_surrogates_deep(dirty) + assert cleaned is not dirty # rebuilt + first = cleaned["content"][0]["text"] + assert "\ud83e" not in first + # Full payload must be UTF-8 encodable. + import json + + json.dumps(cleaned).encode("utf-8") + + def test_paired_surrogates_in_list_collapse_to_emoji(self): + dirty = ["a", "b \ud83e\udd16", "c"] + cleaned = sanitize_surrogates_deep(dirty) + assert cleaned == ["a", "b πŸ€–", "c"] + + def test_deep_recursion_on_tuple_and_dict(self): + dirty = ( + {"k": "\ud83e"}, + ["nested", "\ud83e\udd16"], + "\ud83e clean", + ) + cleaned = sanitize_surrogates_deep(dirty) + # tuple preserved as tuple + assert isinstance(cleaned, tuple) + assert "\ud83e" not in cleaned[0]["k"] + assert cleaned[1][1] == "πŸ€–" + + +class TestProviderSanitizeEmptyContent: + """The provider-boundary defense-in-depth pass. + + Lone surrogates leaking into any string leaf of a request message must + not survive ``_sanitize_empty_content``; otherwise the HTTP client will + raise ``UnicodeEncodeError`` when serializing the request body. + """ + + def test_lone_surrogate_in_string_content_scrubbed(self): + messages = [ + {"role": "user", "content": "leak \ud83e here"}, + ] + result = LLMProvider._sanitize_empty_content(messages) + text = result[0]["content"] + assert "\ud83e" not in text + text.encode("utf-8") # would raise pre-fix + + def test_lone_surrogate_in_content_block_scrubbed(self): + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "hi \ud83e there"}, + ], + }, + ] + result = LLMProvider._sanitize_empty_content(messages) + text = result[0]["content"][0]["text"] + assert "\ud83e" not in text + text.encode("utf-8") + + def test_paired_surrogates_reconstructed_in_content(self): + messages = [ + {"role": "user", "content": "robot \ud83e\udd16 hello"}, + ] + result = LLMProvider._sanitize_empty_content(messages) + assert result[0]["content"] == "robot πŸ€– hello" + + def test_clean_input_semantics_unchanged(self): + messages = [ + {"role": "user", "content": "plain πŸ€– content"}, + {"role": "assistant", "content": [{"type": "text", "text": "ok"}]}, + ] + result = LLMProvider._sanitize_empty_content(messages) + assert result[0]["content"] == "plain πŸ€– content" + assert result[1]["content"][0]["text"] == "ok" + + def test_full_request_body_is_utf8_encodable_after_sanitize(self): + """End-to-end: an emoji-heavy history plus a lone surrogate must + no longer break utf-8 encoding of the outgoing HTTP request body.""" + import json + + messages = [ + {"role": "system", "content": "You are a 🐱 assistant."}, + {"role": "user", "content": "πŸ€– fixed?"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "answer with a lone \ud83e half"}, + ], + }, + ] + cleaned = LLMProvider._sanitize_empty_content(messages) + # This is the operation that previously raised UnicodeEncodeError. + json.dumps({"model": "x", "messages": cleaned}).encode("utf-8") + + +class TestBackwardCompatReExport: + def test_cli_reexport_points_to_shared_helper(self): + """The CLI module continues to expose ``_sanitize_surrogates`` as a + thin alias so existing imports (e.g. ``SafeFileHistory`` in the + legacy tests) keep working.""" + from nanobot.cli.commands import _sanitize_surrogates as cli_alias + + assert cli_alias is sanitize_surrogates + assert cli_alias("hello \ud83e\udd16") == "hello πŸ€–" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"])