fix(cli): sanitize surrogate code points before entering message bus
On Windows, prompt_toolkit produces lone surrogate code points (e.g. 🐈) for emoji input. These propagate through the message bus and crash at json.dumps() / file write time because surrogates cannot be encoded as UTF-8. Extract _sanitize_surrogates() that round-trips through UTF-16 to reconstruct paired surrogates into real characters (e.g. 🐈 → 🐈), replacing unpaired surrogates with U+FFFD. Apply it at the CLI input path and reuse in SafeFileHistory.
This commit is contained in:
@@ -3,12 +3,40 @@
|
||||
Surrogate characters in CLI input must not crash history file writes.
|
||||
"""
|
||||
|
||||
from nanobot.cli.commands import SafeFileHistory
|
||||
from nanobot.cli.commands import SafeFileHistory, _sanitize_surrogates
|
||||
|
||||
|
||||
class TestSanitizeSurrogates:
|
||||
def test_paired_surrogates_reconstructed(self):
|
||||
"""Windows console produces \\ud83d\\udc08 for U+1F408 — must be restored."""
|
||||
result = _sanitize_surrogates("你为什么会用 🐈")
|
||||
assert result == "你为什么会用 🐈"
|
||||
|
||||
def test_lone_surrogates_replaced(self):
|
||||
result = _sanitize_surrogates("hello \udce9 world")
|
||||
assert "\udce9" not in result
|
||||
assert "hello" in result
|
||||
assert "world" in result
|
||||
|
||||
def test_normal_text_unchanged(self):
|
||||
assert _sanitize_surrogates("normal ascii text") == "normal ascii text"
|
||||
|
||||
def test_emoji_already_correct(self):
|
||||
"""Properly encoded emoji should pass through unchanged."""
|
||||
assert _sanitize_surrogates("hello 🐈 nanobot") == "hello 🐈 nanobot"
|
||||
|
||||
def test_mixed_unicode_preserved(self):
|
||||
assert _sanitize_surrogates("你好 hello こんにちは 🎉") == "你好 hello こんにちは 🎉"
|
||||
|
||||
def test_multiple_lone_surrogates(self):
|
||||
result = _sanitize_surrogates("\udce9\udcf1\udcff")
|
||||
assert "\udce9" not in result
|
||||
assert "\udcf1" not in result
|
||||
assert "\udcff" not in result
|
||||
|
||||
|
||||
class TestSafeFileHistory:
|
||||
def test_surrogate_replaced(self, tmp_path):
|
||||
"""Surrogate pairs are replaced with U+FFFD, not crash."""
|
||||
hist = SafeFileHistory(str(tmp_path / "history"))
|
||||
hist.store_string("hello \udce9 world")
|
||||
entries = list(hist.load_history_strings())
|
||||
|
||||
Reference in New Issue
Block a user