fix(providers): sanitize UTF-16 surrogates at provider request boundary

Symptom
-------
LLM requests intermittently fail with:
  'utf-8' codec can't encode characters in position N-N+1: surrogates not allowed
when messages contain emoji-heavy content (e.g. HTML with mixed emoji + JSON round-trips).
This blocks the affected session until the session file is quarantined.

Root cause
----------
Surrogate sanitization was only applied at the CLI entry point
(nanobot/cli/commands.py: _sanitize_surrogates). Requests entering
the LLM provider layer through other channels (Feishu, cron, webui,
tool results, memory injection) had no defensive cleaning, so any
message that happened to carry unpaired UTF-16 surrogates (from an
upstream JSON round-trip with ensure_ascii=True on ill-formed input,
memory rehydration, or third-party content) would blow up at
json.dumps -> HTTP encode time inside the provider client.

Fix
---
1. Extract sanitize_surrogates() and sanitize_surrogates_deep() into
   nanobot/utils/helpers.py as the single source of truth. Both use
   utf-16-le round-tripping with errors='surrogatepass' / 'replace',
   so paired surrogates reconstruct back into their real code point
   and lone surrogates collapse to U+FFFD.
2. Make nanobot/cli/commands.py:_sanitize_surrogates a thin wrapper
   that re-exports the shared helper (backward compatible).
3. Add defense-in-depth at the LLM provider boundary in
   nanobot/providers/base.py:_sanitize_empty_content by running
   sanitize_surrogates_deep over each message and its content blocks
   right before requests are serialized to JSON.

Non-goals
---------
- truncate_text() is intentionally left untouched. Python str slicing
  cannot split a single code point into surrogate halves, so it is
  not the source of lone surrogates.
- session/manager storage layer is untouched. Archived sessions
  reproduced the failure only through the request path, not through
  storage.

Verification
------------
- New regression suite tests/providers/test_sanitize_surrogates.py
  covers: paired surrogate reconstruction, lone surrogate replacement,
  identity return on clean input (zero allocation), deep recursion on
  dict/list/tuple, provider _sanitize_empty_content integration, and
  full utf-8 encodability of the sanitized request body.
- 14/14 new tests pass; full existing test module also green.
- Replayed 58 archived real session messages plus adversarial
  lone-surrogate injection through the provider path with no encode
  errors after the fix.

Impact
------
- No behaviour change for clean inputs (sanitize_surrogates_deep is
  an identity return when no surrogate is present).
- Fails-safe: unpaired surrogates degrade to U+FFFD instead of
  aborting the entire request.

nanobot/cli/commands.py

@@ -78,7 +78,12 @@ from nanobot.config.paths import get_workspace_path, is_default_workspace  # noq
 from nanobot.config.schema import Config  # noqa: E402
 from nanobot.security.network import is_loopback_host  # noqa: E402
 from nanobot.utils.evaluator import evaluate_response, resolve_evaluator_prompt  # noqa: E402
-from nanobot.utils.helpers import sync_workspace_templates  # noqa: E402
+from nanobot.utils.helpers import (  # noqa: E402
+    sanitize_surrogates as _sanitize_surrogates,
+)
+from nanobot.utils.helpers import (  # noqa: E402
+    sync_workspace_templates,
+)
 from nanobot.utils.restart import (  # noqa: E402
     consume_restart_notice_from_env,
     format_restart_completed_message,
@@ -92,17 +97,6 @@ from nanobot.webui.build import (  # noqa: E402
 from nanobot.webui.sidebar_state import read_webui_sidebar_state  # noqa: E402
 
 
-def _sanitize_surrogates(text: str) -> str:
-    """Reconstruct surrogate pairs into real characters; replace lone surrogates.
-
-    On Windows, console input may produce lone surrogate code points (e.g.
-    ``\\ud83d\\udc08`` for U+1F408).  Round-tripping through UTF-16 reconstructs
-    paired surrogates into their actual characters and replaces unpaired ones
-    with U+FFFD.
-    """
-    return text.encode("utf-16-le", errors="surrogatepass").decode("utf-16-le", errors="replace")
-
-
 def _signal_name(signum: int) -> str:
     with suppress(ValueError):
         return signal.Signals(signum).name

nanobot/providers/base.py

@@ -15,6 +15,8 @@ from typing import Any
 import json_repair
 from loguru import logger
 
+from nanobot.utils.helpers import sanitize_surrogates_deep
+
 STREAM_IDLE_TIMEOUT_ENV = "NANOBOT_STREAM_IDLE_TIMEOUT_S"
 DEFAULT_STREAM_IDLE_TIMEOUT_S = 90.0
 MAX_STREAM_IDLE_TIMEOUT_S = 3600.0
@@ -272,7 +274,15 @@ class LLMProvider(ABC):
 
     @staticmethod
     def _sanitize_empty_content(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
-        """Sanitize message content: fix empty blocks, strip internal _meta fields."""
+        """Sanitize message content: fix empty blocks, strip internal _meta fields.
+
+        Also strips unpaired UTF-16 surrogate code points from every string leaf
+        as a defense-in-depth pass before the payload leaves the process. Lone
+        surrogates (e.g. leaking from a Windows console, prompt_toolkit history,
+        or a truncated JSON round-trip) otherwise cause ``UnicodeEncodeError:
+        'utf-8' codec can't encode characters ... surrogates not allowed`` when
+        the HTTP client serializes the request body.
+        """
         result: list[dict[str, Any]] = []
         for raw_msg in messages:
             msg = {key: value for key, value in raw_msg.items() if key != "_meta"}
@@ -318,7 +328,10 @@ class LLMProvider(ABC):
                 continue
 
             result.append(msg)
-        return result
+        # Defense-in-depth: scrub lone UTF-16 surrogates from every string leaf.
+        # This is idempotent and no-op when messages are already clean.
+        sanitized = sanitize_surrogates_deep(result)
+        return sanitized if isinstance(sanitized, list) else result
 
     @staticmethod
     def _tool_name(tool: dict[str, Any]) -> str:

nanobot/utils/helpers.py

@@ -21,6 +21,68 @@ _TOOLS_TOKEN_CACHE_MAX_ENTRIES = 64
 _TOOLS_TOKEN_CACHE: dict[int, tuple[tuple[int, ...], dict[bool, int]]] = {}
 
 
+def sanitize_surrogates(text: str) -> str:
+    """Reconstruct surrogate pairs and replace unpaired surrogates.
+
+    Lone UTF-16 surrogate code points (``U+D800``..``U+DFFF``) cannot be
+    encoded as UTF-8 and cause ``UnicodeEncodeError`` when the message is
+    serialized for an HTTP request body. This helper round-trips through
+    UTF-16 to reconstruct genuine surrogate pairs (produced e.g. by Windows
+    console input for emoji) and substitutes lone surrogates with
+    ``U+FFFD``.
+
+    Non-string inputs are returned unchanged so this helper is safe to call
+    on arbitrary message payload leaves.
+    """
+    if not isinstance(text, str):
+        return text
+    # Fast path: no surrogate code points → return the original object so
+    # callers can rely on identity to detect an actual mutation.
+    for ch in text:
+        cp = ord(ch)
+        if 0xD800 <= cp <= 0xDFFF:
+            break
+    else:
+        return text
+    return text.encode("utf-16-le", errors="surrogatepass").decode(
+        "utf-16-le", errors="replace"
+    )
+
+
+def sanitize_surrogates_deep(value: Any) -> Any:
+    """Recursively apply :func:`sanitize_surrogates` to every string leaf.
+
+    Lists and dicts are rebuilt only when a nested string actually changes,
+    so the common case (no surrogates present) returns the original object
+    without allocations.
+    """
+    if isinstance(value, str):
+        cleaned = sanitize_surrogates(value)
+        return cleaned
+    if isinstance(value, list):
+        result_list: list[Any] = []
+        mutated = False
+        for item in value:
+            new_item = sanitize_surrogates_deep(item)
+            if new_item is not item:
+                mutated = True
+            result_list.append(new_item)
+        return result_list if mutated else value
+    if isinstance(value, dict):
+        result_dict: dict[Any, Any] = {}
+        mutated = False
+        for key, item in value.items():
+            new_item = sanitize_surrogates_deep(item)
+            if new_item is not item:
+                mutated = True
+            result_dict[key] = new_item
+        return result_dict if mutated else value
+    if isinstance(value, tuple):
+        result_tuple = tuple(sanitize_surrogates_deep(item) for item in value)
+        return result_tuple if any(a is not b for a, b in zip(result_tuple, value)) else value
+    return value
+
+
 @lru_cache(maxsize=1)
 def _get_token_encoding() -> Any:
     return tiktoken.get_encoding("cl100k_base")

tests/providers/test_sanitize_surrogates.py

@@ -0,0 +1,174 @@
+"""Regression tests for lone UTF-16 surrogate scrubbing in provider payloads.
+
+These lock down two behaviors:
+
+1. ``nanobot.utils.helpers.sanitize_surrogates`` / ``sanitize_surrogates_deep``
+   produce strings that can round-trip through ``str.encode('utf-8')`` even
+   when the input contains unpaired surrogates.
+
+2. ``LLMProvider._sanitize_empty_content`` applies the deep sanitize as a
+   defense-in-depth pass so that ``UnicodeEncodeError: 'utf-8' codec can't
+   encode characters ... surrogates not allowed`` cannot escape into the
+   HTTP client when messages contain emoji-heavy history plus a lone
+   surrogate leak (e.g. from Windows console input or a truncated JSON
+   round-trip).
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from nanobot.providers.base import LLMProvider
+from nanobot.utils.helpers import (
+    sanitize_surrogates,
+    sanitize_surrogates_deep,
+)
+
+
+class TestSanitizeSurrogates:
+    def test_paired_surrogates_reconstructed_to_emoji(self):
+        # \uD83E\uDD16 is the UTF-16 surrogate pair for 🤖 (U+1F916).
+        raw = "hello \ud83e\udd16 world"
+        cleaned = sanitize_surrogates(raw)
+        assert cleaned == "hello 🤖 world"
+        cleaned.encode("utf-8")  # must not raise
+
+    def test_lone_surrogate_replaced_with_fffd(self):
+        raw = "hello \ud83e world"  # lone high surrogate
+        cleaned = sanitize_surrogates(raw)
+        assert "\ud83e" not in cleaned
+        # replacement char U+FFFD substitutes the unpaired surrogate
+        assert "\ufffd" in cleaned
+        cleaned.encode("utf-8")  # must not raise
+
+    def test_normal_text_returned_unchanged_identity(self):
+        raw = "plain ascii + 中文 + 🤖"
+        assert sanitize_surrogates(raw) == raw
+
+    def test_non_string_returned_as_is(self):
+        assert sanitize_surrogates(None) is None  # type: ignore[arg-type]
+        assert sanitize_surrogates(42) == 42  # type: ignore[arg-type]
+
+
+class TestSanitizeSurrogatesDeep:
+    def test_clean_input_returns_same_object(self):
+        payload = {"role": "user", "content": [{"type": "text", "text": "hi 🤖"}]}
+        result = sanitize_surrogates_deep(payload)
+        # No allocation on clean input: same object identity.
+        assert result is payload
+
+    def test_lone_surrogate_in_nested_content_cleaned(self):
+        dirty = {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "hello \ud83e world"},
+                {"type": "text", "text": "normal"},
+            ],
+        }
+        cleaned = sanitize_surrogates_deep(dirty)
+        assert cleaned is not dirty  # rebuilt
+        first = cleaned["content"][0]["text"]
+        assert "\ud83e" not in first
+        # Full payload must be UTF-8 encodable.
+        import json
+
+        json.dumps(cleaned).encode("utf-8")
+
+    def test_paired_surrogates_in_list_collapse_to_emoji(self):
+        dirty = ["a", "b \ud83e\udd16", "c"]
+        cleaned = sanitize_surrogates_deep(dirty)
+        assert cleaned == ["a", "b 🤖", "c"]
+
+    def test_deep_recursion_on_tuple_and_dict(self):
+        dirty = (
+            {"k": "\ud83e"},
+            ["nested", "\ud83e\udd16"],
+            "\ud83e clean",
+        )
+        cleaned = sanitize_surrogates_deep(dirty)
+        # tuple preserved as tuple
+        assert isinstance(cleaned, tuple)
+        assert "\ud83e" not in cleaned[0]["k"]
+        assert cleaned[1][1] == "🤖"
+
+
+class TestProviderSanitizeEmptyContent:
+    """The provider-boundary defense-in-depth pass.
+
+    Lone surrogates leaking into any string leaf of a request message must
+    not survive ``_sanitize_empty_content``; otherwise the HTTP client will
+    raise ``UnicodeEncodeError`` when serializing the request body.
+    """
+
+    def test_lone_surrogate_in_string_content_scrubbed(self):
+        messages = [
+            {"role": "user", "content": "leak \ud83e here"},
+        ]
+        result = LLMProvider._sanitize_empty_content(messages)
+        text = result[0]["content"]
+        assert "\ud83e" not in text
+        text.encode("utf-8")  # would raise pre-fix
+
+    def test_lone_surrogate_in_content_block_scrubbed(self):
+        messages = [
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": "hi \ud83e there"},
+                ],
+            },
+        ]
+        result = LLMProvider._sanitize_empty_content(messages)
+        text = result[0]["content"][0]["text"]
+        assert "\ud83e" not in text
+        text.encode("utf-8")
+
+    def test_paired_surrogates_reconstructed_in_content(self):
+        messages = [
+            {"role": "user", "content": "robot \ud83e\udd16 hello"},
+        ]
+        result = LLMProvider._sanitize_empty_content(messages)
+        assert result[0]["content"] == "robot 🤖 hello"
+
+    def test_clean_input_semantics_unchanged(self):
+        messages = [
+            {"role": "user", "content": "plain 🤖 content"},
+            {"role": "assistant", "content": [{"type": "text", "text": "ok"}]},
+        ]
+        result = LLMProvider._sanitize_empty_content(messages)
+        assert result[0]["content"] == "plain 🤖 content"
+        assert result[1]["content"][0]["text"] == "ok"
+
+    def test_full_request_body_is_utf8_encodable_after_sanitize(self):
+        """End-to-end: an emoji-heavy history plus a lone surrogate must
+        no longer break utf-8 encoding of the outgoing HTTP request body."""
+        import json
+
+        messages = [
+            {"role": "system", "content": "You are a 🐱 assistant."},
+            {"role": "user", "content": "🤖 fixed?"},
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": "answer with a lone \ud83e half"},
+                ],
+            },
+        ]
+        cleaned = LLMProvider._sanitize_empty_content(messages)
+        # This is the operation that previously raised UnicodeEncodeError.
+        json.dumps({"model": "x", "messages": cleaned}).encode("utf-8")
+
+
+class TestBackwardCompatReExport:
+    def test_cli_reexport_points_to_shared_helper(self):
+        """The CLI module continues to expose ``_sanitize_surrogates`` as a
+        thin alias so existing imports (e.g. ``SafeFileHistory`` in the
+        legacy tests) keep working."""
+        from nanobot.cli.commands import _sanitize_surrogates as cli_alias
+
+        assert cli_alias is sanitize_surrogates
+        assert cli_alias("hello \ud83e\udd16") == "hello 🤖"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])