feat(webui): support image uploads in composer and message bubbles

This commit is contained in:
Xubin Ren
2026-04-23 00:07:27 +08:00
committed by Xubin Ren
parent c1e7aa5504
commit 61a28c2c0a
39 changed files with 3670 additions and 124 deletions
+10 -8
View File
@@ -858,15 +858,17 @@ class AgentLoop:
)
)
# Persist the triggering user message immediately, before running the
# agent loop. If the process is killed mid-turn (OOM, SIGKILL, self-
# restart, etc.), the existing runtime_checkpoint preserves the
# in-flight assistant/tool state but NOT the user message itself, so
# the user's prompt is silently lost on recovery. Saving it up front
# makes recovery possible from the session log alone.
# Persist the triggering user message up front so a mid-turn crash
# doesn't silently lose the prompt on recovery. ``media`` rides along
# as raw on-disk paths — sanitized image blocks are stripped from
# JSONL, and webui replay needs the paths to mint signed URLs.
user_persisted_early = False
if isinstance(msg.content, str) and msg.content.strip():
session.add_message("user", msg.content)
media_paths = [p for p in (msg.media or []) if isinstance(p, str) and p]
has_text = isinstance(msg.content, str) and msg.content.strip()
if has_text or media_paths:
extra: dict[str, Any] = {"media": list(media_paths)} if media_paths else {}
text = msg.content if isinstance(msg.content, str) else ""
session.add_message("user", text, **extra)
self._mark_pending_user_turn(session)
self.sessions.save(session)
user_persisted_early = True
+12 -29
View File
@@ -7,13 +7,9 @@ All requests route to a single persistent API session.
from __future__ import annotations
import asyncio
import base64
import json as _json
import mimetypes
import re
import time
import uuid
from pathlib import Path
from typing import Any
from aiohttp import web
@@ -21,14 +17,20 @@ from loguru import logger
from nanobot.config.paths import get_media_dir
from nanobot.utils.helpers import safe_filename
from nanobot.utils.media_decode import (
FileSizeExceeded as _FileSizeExceeded,
MAX_FILE_SIZE,
save_base64_data_url as _save_base64_data_url,
)
from nanobot.utils.runtime import EMPTY_FINAL_RESPONSE_MESSAGE
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MB
_DATA_URL_RE = re.compile(r"^data:([^;]+);base64,(.+)$", re.DOTALL)
class _FileSizeExceeded(Exception):
"""Raised when an uploaded file exceeds the size limit."""
__all__ = (
"MAX_FILE_SIZE",
"_FileSizeExceeded",
"_save_base64_data_url",
"create_app",
"handle_chat_completions",
)
API_SESSION_KEY = "api:default"
@@ -102,25 +104,6 @@ _SSE_DONE = b"data: [DONE]\n\n"
# ---------------------------------------------------------------------------
def _save_base64_data_url(data_url: str, media_dir: Path) -> str | None:
"""Decode a data:...;base64,... URL and save to disk."""
m = _DATA_URL_RE.match(data_url)
if not m:
return None
mime_type, b64_payload = m.group(1), m.group(2)
try:
raw = base64.b64decode(b64_payload)
except Exception:
return None
if len(raw) > MAX_FILE_SIZE:
raise _FileSizeExceeded(f"File exceeds {MAX_FILE_SIZE // (1024 * 1024)}MB limit")
ext = mimetypes.guess_extension(mime_type) or ".bin"
filename = f"{uuid.uuid4().hex[:12]}{ext}"
dest = media_dir / safe_filename(filename)
dest.write_bytes(raw)
return str(dest)
def _parse_json_content(body: dict) -> tuple[str, list[str]]:
"""Parse JSON request body. Returns (text, media_paths)."""
messages = body.get("messages")
+246 -2
View File
@@ -3,7 +3,10 @@
from __future__ import annotations
import asyncio
import base64
import binascii
import email.utils
import hashlib
import hmac
import http
import json
@@ -28,7 +31,12 @@ from websockets.http11 import Response
from nanobot.bus.events import OutboundMessage
from nanobot.bus.queue import MessageBus
from nanobot.channels.base import BaseChannel
from nanobot.config.paths import get_media_dir
from nanobot.config.schema import Base
from nanobot.utils.media_decode import (
FileSizeExceeded,
save_base64_data_url,
)
if TYPE_CHECKING:
from nanobot.session.manager import SessionManager
@@ -75,7 +83,11 @@ class WebSocketConfig(Base):
websocket_requires_token: bool = True
allow_from: list[str] = Field(default_factory=lambda: ["*"])
streaming: bool = True
max_message_bytes: int = Field(default=1_048_576, ge=1024, le=16_777_216)
# Default 36 MB, upper 40 MB: supports up to 4 images at ~6 MB each after
# client-side Worker normalization (see webui Composer). 4 × 6 MB × 1.37
# (base64 overhead) + envelope framing stays under 36 MB; the 40 MB ceiling
# leaves a small margin for sender slop without opening a DoS avenue.
max_message_bytes: int = Field(default=37_748_736, ge=1024, le=41_943_040)
ping_interval_s: float = Field(default=20.0, ge=5.0, le=300.0)
ping_timeout_s: float = Field(default=20.0, ge=5.0, le=300.0)
ssl_certfile: str = ""
@@ -206,6 +218,35 @@ def _parse_envelope(raw: str) -> dict[str, Any] | None:
return data
# Per-message image limits. The server-side guard is a touch looser than the
# client's ``Worker`` normalization target (6 MB) — tolerate client slop, but
# still cap total ingress at ``_MAX_IMAGES_PER_MESSAGE * _MAX_IMAGE_BYTES``
# which fits comfortably inside ``max_message_bytes``.
_MAX_IMAGES_PER_MESSAGE = 4
_MAX_IMAGE_BYTES = 8 * 1024 * 1024
# Image MIME whitelist — matches the Composer's ``accept`` list. SVG is
# explicitly excluded to avoid the XSS surface inside embedded scripts.
_IMAGE_MIME_ALLOWED: frozenset[str] = frozenset({
"image/png",
"image/jpeg",
"image/webp",
"image/gif",
})
_DATA_URL_MIME_RE = re.compile(r"^data:([^;]+);base64,", re.DOTALL)
def _extract_data_url_mime(url: str) -> str | None:
"""Return the MIME type of a ``data:<mime>;base64,...`` URL, else ``None``."""
if not isinstance(url, str):
return None
m = _DATA_URL_MIME_RE.match(url)
if not m:
return None
return m.group(1).strip().lower() or None
_LOCALHOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
# Matches the legacy chat-id pattern but allows file-system-safe stems too,
@@ -278,6 +319,29 @@ def _is_websocket_upgrade(request: WsRequest) -> bool:
return True
def _b64url_encode(data: bytes) -> str:
"""URL-safe base64 without padding — compact + friendly in URL paths."""
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
def _b64url_decode(s: str) -> bytes:
"""Reverse of :func:`_b64url_encode`; caller handles ``ValueError``."""
pad = "=" * (-len(s) % 4)
return base64.urlsafe_b64decode(s + pad)
# Allowed MIME types we actually serve from the media endpoint. Anything
# outside this set is degraded to ``application/octet-stream`` so an
# attacker who somehow gets a signed URL for an unexpected file type can't
# trick the browser into sniffing executable content.
_MEDIA_ALLOWED_MIMES: frozenset[str] = frozenset({
"image/png",
"image/jpeg",
"image/webp",
"image/gif",
})
def _issue_route_secret_matches(headers: Any, configured_secret: str) -> bool:
"""Return True if the token-issue HTTP request carries credentials matching ``token_issue_secret``."""
if not configured_secret:
@@ -326,6 +390,11 @@ class WebSocketChannel(BaseChannel):
self._static_dist_path: Path | None = (
static_dist_path.resolve() if static_dist_path is not None else None
)
# Process-local secret used to HMAC-sign media URLs. The signed URL is
# the capability — anyone who holds a valid URL can fetch that one
# file, nothing else. The secret regenerates on restart so links
# become self-expiring (callers just refresh the session list).
self._media_secret: bytes = secrets.token_bytes(32)
# -- Subscription bookkeeping -------------------------------------------
@@ -457,6 +526,14 @@ class WebSocketChannel(BaseChannel):
if m:
return self._handle_session_delete(request, m.group(1))
# Signed media fetch: ``<sig>`` is an HMAC over ``<payload>``; the
# payload decodes to a path inside :func:`get_media_dir`. See
# :meth:`_sign_media_path` for the inverse direction used to build
# these URLs when replaying a session.
m = re.match(r"^/api/media/([A-Za-z0-9_-]+)/([A-Za-z0-9_-]+)$", got)
if m:
return self._handle_media_fetch(m.group(1), m.group(2))
# 4. WebSocket upgrade (the channel's primary purpose). Only run the
# handshake gate on requests that actually ask to upgrade; otherwise
# a bare ``GET /`` from the browser would be rejected as an
@@ -568,8 +645,109 @@ class WebSocketChannel(BaseChannel):
data = self._session_manager.read_session_file(decoded_key)
if data is None:
return _http_error(404, "session not found")
# Decorate persisted user messages with signed media URLs so the
# client can render previews. The raw on-disk ``media`` paths are
# stripped on the way out — they leak server filesystem layout and
# the client never needs them once it has the signed fetch URL.
self._augment_media_urls(data)
return _http_json_response(data)
def _augment_media_urls(self, payload: dict[str, Any]) -> None:
"""Mutate *payload* in place: each message's ``media`` path list is
replaced by a parallel ``media_urls`` list of signed fetch URLs.
Messages without media or with non-string path entries are left
untouched. Paths that no longer live inside ``media_dir`` (e.g. the
file was deleted, or the dir was relocated) are silently skipped;
the client falls back to the historical-replay placeholder tile.
"""
messages = payload.get("messages")
if not isinstance(messages, list):
return
for msg in messages:
if not isinstance(msg, dict):
continue
media = msg.get("media")
if not isinstance(media, list) or not media:
continue
urls: list[dict[str, str]] = []
for entry in media:
if not isinstance(entry, str) or not entry:
continue
signed = self._sign_media_path(Path(entry))
if signed is None:
continue
urls.append({"url": signed, "name": Path(entry).name})
if urls:
msg["media_urls"] = urls
# Always drop the raw paths from the wire payload.
msg.pop("media", None)
def _sign_media_path(self, abs_path: Path) -> str | None:
"""Return a ``/api/media/<sig>/<payload>`` URL for *abs_path*, or
``None`` when the path does not resolve inside the media root.
The URL is self-authenticating: the signature binds the payload to
this process's ``_media_secret``, so only paths we chose to sign can
be fetched. The returned path is relative to the server origin; the
client joins it against the existing webui base.
"""
try:
media_root = get_media_dir().resolve()
rel = abs_path.resolve().relative_to(media_root)
except (OSError, ValueError):
return None
payload = _b64url_encode(rel.as_posix().encode("utf-8"))
mac = hmac.new(
self._media_secret, payload.encode("ascii"), hashlib.sha256
).digest()[:16]
return f"/api/media/{_b64url_encode(mac)}/{payload}"
def _handle_media_fetch(self, sig: str, payload: str) -> Response:
"""Serve a single media file previously signed via
:meth:`_sign_media_path`. Validates the signature, decodes the
payload to a relative path, and streams the file bytes with a
long-lived immutable cache header (the URL already encodes the
file identity, so caches can be aggressive)."""
try:
provided_mac = _b64url_decode(sig)
except (ValueError, binascii.Error):
return _http_error(401, "invalid signature")
expected_mac = hmac.new(
self._media_secret, payload.encode("ascii"), hashlib.sha256
).digest()[:16]
if not hmac.compare_digest(expected_mac, provided_mac):
return _http_error(401, "invalid signature")
try:
rel_bytes = _b64url_decode(payload)
rel_str = rel_bytes.decode("utf-8")
except (ValueError, binascii.Error, UnicodeDecodeError):
return _http_error(400, "invalid payload")
# An attacker who somehow bypassed the HMAC check would still need
# the resolved path to escape the media root; guard defensively.
try:
media_root = get_media_dir().resolve()
candidate = (media_root / rel_str).resolve()
candidate.relative_to(media_root)
except (OSError, ValueError):
return _http_error(404, "not found")
if not candidate.is_file():
return _http_error(404, "not found")
try:
body = candidate.read_bytes()
except OSError:
return _http_error(500, "read error")
mime, _ = mimetypes.guess_type(candidate.name)
if mime not in _MEDIA_ALLOWED_MIMES:
mime = "application/octet-stream"
return _http_response(
body,
content_type=mime,
extra_headers=[
("Cache-Control", "private, max-age=31536000, immutable"),
],
)
def _handle_session_delete(self, request: WsRequest, key: str) -> Response:
if not self._check_api_token(request):
return _http_error(401, "Unauthorized")
@@ -755,6 +933,48 @@ class WebSocketChannel(BaseChannel):
finally:
self._cleanup_connection(connection)
@staticmethod
def _save_envelope_media(
media: list[Any],
) -> tuple[list[str], str | None]:
"""Decode and persist ``media`` items from a ``message`` envelope.
Returns ``(paths, None)`` on success or ``([], reason)`` on the first
failure — the caller is expected to surface ``reason`` to the client
and skip publishing so no half-formed message ever reaches the agent.
``reason`` is a short, stable token suitable for UI localization.
Shape: ``list[{"data_url": str, "name"?: str | None}]``.
"""
if len(media) > _MAX_IMAGES_PER_MESSAGE:
return [], "too_many_images"
media_dir = get_media_dir("websocket")
paths: list[str] = []
for item in media:
if not isinstance(item, dict):
return [], "malformed"
data_url = item.get("data_url")
if not isinstance(data_url, str) or not data_url:
return [], "malformed"
mime = _extract_data_url_mime(data_url)
if mime is None:
return [], "decode"
if mime not in _IMAGE_MIME_ALLOWED:
return [], "mime"
try:
saved = save_base64_data_url(
data_url, media_dir, max_bytes=_MAX_IMAGE_BYTES,
)
except FileSizeExceeded:
return [], "size"
except Exception as exc:
logger.warning("websocket: media decode failed: {}", exc)
return [], "decode"
if saved is None:
return [], "decode"
paths.append(saved)
return paths, None
async def _dispatch_envelope(
self,
connection: Any,
@@ -782,15 +1002,39 @@ class WebSocketChannel(BaseChannel):
if not _is_valid_chat_id(cid):
await self._send_event(connection, "error", detail="invalid chat_id")
return
if not isinstance(content, str) or not content.strip():
if not isinstance(content, str):
await self._send_event(connection, "error", detail="missing content")
return
raw_media = envelope.get("media")
media_paths: list[str] = []
if raw_media is not None:
if not isinstance(raw_media, list):
await self._send_event(
connection, "error",
detail="image_rejected", reason="malformed",
)
return
media_paths, reason = self._save_envelope_media(raw_media)
if reason is not None:
await self._send_event(
connection, "error",
detail="image_rejected", reason=reason,
)
return
# Allow image-only turns (content may be empty when media is attached).
if not content.strip() and not media_paths:
await self._send_event(connection, "error", detail="missing content")
return
# Auto-attach on first use so clients can one-shot without a separate attach.
self._attach(connection, cid)
await self._handle_message(
sender_id=client_id,
chat_id=cid,
content=content,
media=media_paths or None,
metadata={"remote": getattr(connection, "remote_address", None)},
)
return
+19 -2
View File
@@ -11,7 +11,12 @@ from typing import Any
from loguru import logger
from nanobot.config.paths import get_legacy_sessions_dir
from nanobot.utils.helpers import ensure_dir, find_legal_message_start, safe_filename
from nanobot.utils.helpers import (
ensure_dir,
find_legal_message_start,
image_placeholder_text,
safe_filename,
)
@dataclass
@@ -54,7 +59,19 @@ class Session:
out: list[dict[str, Any]] = []
for message in sliced:
entry: dict[str, Any] = {"role": message["role"], "content": message.get("content", "")}
content = message.get("content", "")
# Synthesize an ``[image: path]`` breadcrumb from the persisted
# ``media`` kwarg so LLM replay still sees *something* where the
# image used to be. Without this, an image-only user turn
# replays as an empty user message — the assistant's reply then
# looks like it's responding to nothing.
media = message.get("media")
if isinstance(media, list) and media and isinstance(content, str):
breadcrumbs = "\n".join(
image_placeholder_text(p) for p in media if isinstance(p, str) and p
)
content = f"{content}\n{breadcrumbs}" if content else breadcrumbs
entry: dict[str, Any] = {"role": message["role"], "content": content}
for key in ("tool_calls", "tool_call_id", "name", "reasoning_content"):
if key in message:
entry[key] = message[key]
+55
View File
@@ -0,0 +1,55 @@
"""Shared helpers for decoding ``data:...;base64,...`` URLs to disk.
Historically lived in ``nanobot.api.server``; now shared by the WebSocket
channel so the ``api`` + ``websocket`` ingress paths apply the same parsing,
size guard, and filesystem layout.
"""
from __future__ import annotations
import base64
import mimetypes
import re
import uuid
from pathlib import Path
from nanobot.utils.helpers import safe_filename
DEFAULT_MAX_BYTES = 10 * 1024 * 1024
MAX_FILE_SIZE = DEFAULT_MAX_BYTES
_DATA_URL_RE = re.compile(r"^data:([^;]+);base64,(.+)$", re.DOTALL)
class FileSizeExceeded(Exception):
"""Raised when a decoded payload exceeds the caller's size limit."""
def save_base64_data_url(
data_url: str,
media_dir: Path,
*,
max_bytes: int | None = None,
) -> str | None:
"""Decode a ``data:<mime>;base64,<payload>`` URL and persist it.
Returns the absolute path on success, ``None`` when the URL shape or the
base64 payload itself is malformed. Raises :class:`FileSizeExceeded`
when the decoded payload is larger than ``max_bytes`` (default 10 MB).
"""
m = _DATA_URL_RE.match(data_url)
if not m:
return None
mime_type, b64_payload = m.group(1), m.group(2)
try:
raw = base64.b64decode(b64_payload)
except Exception:
return None
limit = DEFAULT_MAX_BYTES if max_bytes is None else max_bytes
if len(raw) > limit:
raise FileSizeExceeded(f"File exceeds {limit // (1024 * 1024)}MB limit")
ext = mimetypes.guess_extension(mime_type) or ".bin"
filename = f"{uuid.uuid4().hex[:12]}{ext}"
dest = media_dir / safe_filename(filename)
dest.write_bytes(raw)
return str(dest)
+81
View File
@@ -234,6 +234,87 @@ async def test_process_message_persists_user_message_before_turn_completes(tmp_p
assert persisted.updated_at >= persisted.created_at
# 1x1 PNG used by the media-persistence tests. ``extract_documents`` runs
# at the top of ``_process_message`` and filters ``msg.media`` down to
# paths that magic-byte-sniff as images, so the test fixture needs real
# bytes on disk (not just placeholder paths).
_PNG_1X1 = (
b"\x89PNG\r\n\x1a\n"
b"\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01"
b"\x08\x06\x00\x00\x00\x1f\x15\xc4\x89"
b"\x00\x00\x00\nIDATx\x9cc\x00\x00\x00\x02\x00\x01"
b"\x00\x00\x00\x00IEND\xaeB`\x82"
)
@pytest.mark.asyncio
async def test_process_message_persists_media_paths_on_user_turn(tmp_path: Path) -> None:
"""User turns that attach images must record the media paths alongside
the text so the webui can rehydrate previews on session replay.
This is the producer half of the signed-media-URL round-trip: paths are
stored here, then :meth:`WebSocketChannel._augment_media_urls` maps them
onto signed URLs on the way out.
"""
img_a = tmp_path / "uuid-1.png"
img_a.write_bytes(_PNG_1X1)
img_b = tmp_path / "uuid-2.png"
img_b.write_bytes(_PNG_1X1)
loop = _make_full_loop(tmp_path)
loop.consolidator.maybe_consolidate_by_tokens = AsyncMock(return_value=False) # type: ignore[method-assign]
loop._run_agent_loop = AsyncMock(side_effect=RuntimeError("interrupt")) # type: ignore[method-assign]
msg = InboundMessage(
channel="websocket",
sender_id="u1",
chat_id="c-media",
content="look",
media=[str(img_a), str(img_b)],
)
with pytest.raises(RuntimeError, match="interrupt"):
await loop._process_message(msg)
loop.sessions.invalidate("websocket:c-media")
persisted = loop.sessions.get_or_create("websocket:c-media")
assert [m["role"] for m in persisted.messages] == ["user"]
assert persisted.messages[0]["content"] == "look"
assert persisted.messages[0]["media"] == [str(img_a), str(img_b)]
@pytest.mark.asyncio
async def test_process_message_persists_media_only_turn_without_text(tmp_path: Path) -> None:
"""A turn with images but no text still persists (previously silent-dropped).
The old early-persist gate skipped messages without text, leaving pure
image turns un-checkpointed. They now materialise as an empty-content
user row with ``media`` attached.
"""
img = tmp_path / "only.png"
img.write_bytes(_PNG_1X1)
loop = _make_full_loop(tmp_path)
loop.consolidator.maybe_consolidate_by_tokens = AsyncMock(return_value=False) # type: ignore[method-assign]
loop._run_agent_loop = AsyncMock(side_effect=RuntimeError("boom")) # type: ignore[method-assign]
msg = InboundMessage(
channel="websocket",
sender_id="u1",
chat_id="c-images-only",
content="",
media=[str(img)],
)
with pytest.raises(RuntimeError):
await loop._process_message(msg)
loop.sessions.invalidate("websocket:c-images-only")
persisted = loop.sessions.get_or_create("websocket:c-images-only")
assert len(persisted.messages) == 1
assert persisted.messages[0]["role"] == "user"
assert persisted.messages[0]["content"] == ""
assert persisted.messages[0]["media"] == [str(img)]
@pytest.mark.asyncio
async def test_process_message_does_not_duplicate_early_persisted_user_message(tmp_path: Path) -> None:
loop = _make_full_loop(tmp_path)
@@ -217,3 +217,55 @@ def test_window_cuts_mid_tool_group():
# leaving orphan tool results for split_a at the front.
history = session.get_history(max_messages=6)
_assert_no_orphans(history)
# --- Image breadcrumbs: media kwarg is synthesized into content for replay ---
def test_get_history_synthesizes_image_breadcrumb_from_media_kwarg():
"""Persisted user turns carry image paths as a ``media`` kwarg; LLM
replay must still see an ``[image: path]`` breadcrumb so the assistant's
follow-up reply has a referent instead of trailing an empty user row."""
session = Session(key="test:media")
session.messages.append(
{"role": "user", "content": "look", "media": ["/m/a.png", "/m/b.png"]}
)
session.messages.append({"role": "assistant", "content": "nice"})
history = session.get_history(max_messages=500)
assert history == [
{"role": "user", "content": "look\n[image: /m/a.png]\n[image: /m/b.png]"},
{"role": "assistant", "content": "nice"},
]
def test_get_history_synthesizes_breadcrumb_for_image_only_turn():
"""Turns with no text but attached images must not replay as empty
strings — the LLM would otherwise see a bare user turn followed by an
unexplained assistant answer."""
session = Session(key="test:image-only")
session.messages.append({"role": "user", "content": "", "media": ["/m/pic.png"]})
session.messages.append({"role": "assistant", "content": "I see a cat"})
history = session.get_history(max_messages=500)
assert history[0] == {"role": "user", "content": "[image: /m/pic.png]"}
def test_get_history_ignores_media_kwarg_on_non_user_rows():
"""``media`` only ever appears on user entries in practice, but the
synthesizer must be defensive: assistants / tools with list content
don't get the breadcrumb pasted on top."""
session = Session(key="test:defensive")
session.messages.append(
{
"role": "assistant",
"content": [{"type": "text", "text": "structured"}],
"media": ["/m/x.png"], # nonsense but shouldn't crash
}
)
history = session.get_history(max_messages=500)
# List content is passed through verbatim — the synthesizer only
# rewrites plain-string content.
assert history[0]["content"] == [{"type": "text", "text": "structured"}]
@@ -0,0 +1,416 @@
"""Tests for WS envelope media handling (client image upload path).
Exercises ``WebSocketChannel._dispatch_envelope`` for the ``message`` branch:
decoding base64 data URLs, rejecting malformed / oversized / non-whitelisted
payloads, preserving backward compatibility with media-less frames, and
forwarding saved paths to ``_handle_message``.
"""
from __future__ import annotations
import base64
import json
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from nanobot.channels.websocket import (
WebSocketChannel,
_extract_data_url_mime,
)
def _tiny_png_data_url() -> str:
"""A 1-pixel PNG prefixed as a data URL — just enough for magic-bytes sniffing."""
# 1x1 transparent PNG
png = (
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00"
b"\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx"
b"\x9cc\xf8\xcf\xc0\x00\x00\x00\x03\x00\x01\x00\x18\xdd\x8d\xb4\x00"
b"\x00\x00\x00IEND\xaeB`\x82"
)
return f"data:image/png;base64,{base64.b64encode(png).decode()}"
def _data_url(mime: str, payload: bytes) -> str:
return f"data:{mime};base64,{base64.b64encode(payload).decode()}"
def _make_channel() -> WebSocketChannel:
bus = MagicMock()
bus.publish_inbound = AsyncMock()
channel = WebSocketChannel(
{"enabled": True, "allowFrom": ["*"], "websocketRequiresToken": False},
bus,
)
channel._handle_message = AsyncMock() # type: ignore[method-assign]
return channel
# -- Pure helpers --------------------------------------------------------------
@pytest.mark.parametrize(
("url", "expected"),
[
("data:image/png;base64,AAAA", "image/png"),
("data:image/jpeg;base64,AAAA", "image/jpeg"),
("data:IMAGE/PNG;base64,AAAA", "image/png"),
("data:image/svg+xml;base64,AAAA", "image/svg+xml"),
("data:text/plain;base64,AAAA", "text/plain"),
("http://evil.example/x.png", None),
("data:image/png,AAAA", None), # missing `;base64`
("", None),
(None, None),
],
)
def test_extract_data_url_mime(url: Any, expected: str | None) -> None:
assert _extract_data_url_mime(url) == expected
# -- max_message_bytes bump ----------------------------------------------------
def test_max_message_bytes_default_supports_multi_image_frame() -> None:
"""Default 36 MB must comfortably hold 4 × 6 MB base64-encoded images."""
from nanobot.channels.websocket import WebSocketConfig
default = WebSocketConfig().max_message_bytes
# 4 images × 6 MB × 1.37 base64 overhead ≈ 33 MB
assert default >= 33 * 1024 * 1024
# Upper bound 40 MB matches plan
with pytest.raises(Exception):
WebSocketConfig(max_message_bytes=41_943_040 + 1)
# -- _dispatch_envelope message branch + media --------------------------------
@pytest.mark.asyncio
async def test_message_without_media_backward_compatible() -> None:
"""Existing clients that don't send ``media`` keep working unchanged."""
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {"type": "message", "chat_id": "abc123", "content": "hello"}
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_awaited_once()
call = channel._handle_message.call_args
assert call.kwargs["chat_id"] == "abc123"
assert call.kwargs["content"] == "hello"
# When no media, we pass ``media=None`` so downstream treats it as absent.
assert call.kwargs["media"] is None
@pytest.mark.asyncio
async def test_message_with_single_image_forwards_saved_path(tmp_path) -> None:
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "look at this",
"media": [{"data_url": _tiny_png_data_url(), "name": "shot.png"}],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_awaited_once()
paths = channel._handle_message.call_args.kwargs["media"]
assert isinstance(paths, list) and len(paths) == 1
saved = Path(paths[0])
assert saved.exists()
assert saved.suffix == ".png"
assert saved.is_relative_to(tmp_path)
@pytest.mark.asyncio
async def test_message_with_multiple_images(tmp_path) -> None:
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "a couple",
"media": [
{"data_url": _tiny_png_data_url()},
{"data_url": _tiny_png_data_url()},
{"data_url": _tiny_png_data_url()},
],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
paths = channel._handle_message.call_args.kwargs["media"]
assert len(paths) == 3
# Saved filenames must be unique.
assert len({Path(p).name for p in paths}) == 3
@pytest.mark.asyncio
async def test_image_only_message_allows_empty_text(tmp_path) -> None:
"""When media is attached, empty text is acceptable."""
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "",
"media": [{"data_url": _tiny_png_data_url()}],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_awaited_once()
# Error event NOT sent.
mock_conn.send.assert_not_awaited()
@pytest.mark.asyncio
async def test_message_rejected_when_more_than_four_images(tmp_path) -> None:
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "hi",
"media": [{"data_url": _tiny_png_data_url()}] * 5,
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
mock_conn.send.assert_awaited_once()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["event"] == "error"
assert err["detail"] == "image_rejected"
assert err["reason"] == "too_many_images"
@pytest.mark.asyncio
async def test_message_rejected_on_oversize_payload(tmp_path) -> None:
channel = _make_channel()
mock_conn = AsyncMock()
oversized = b"x" * (9 * 1024 * 1024) # > 8 MB WS limit
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "big",
"media": [{"data_url": _data_url("image/png", oversized)}],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["detail"] == "image_rejected"
assert err["reason"] == "size"
@pytest.mark.asyncio
async def test_message_rejected_on_non_image_mime(tmp_path) -> None:
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "pdf?",
"media": [{"data_url": _data_url("application/pdf", b"%PDF-1.4")}],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["detail"] == "image_rejected"
assert err["reason"] == "mime"
@pytest.mark.asyncio
async def test_message_rejected_on_svg_mime(tmp_path) -> None:
"""SVG is explicitly rejected — XSS surface inside embedded scripts."""
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "svg",
"media": [{"data_url": _data_url("image/svg+xml", b"<svg/>")}],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["reason"] == "mime"
@pytest.mark.asyncio
async def test_message_rejected_on_malformed_data_url(tmp_path) -> None:
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "nope",
"media": [{"data_url": "http://evil.example/image.png"}],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["reason"] == "decode"
@pytest.mark.asyncio
async def test_message_rejected_on_broken_base64(tmp_path) -> None:
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "nope",
"media": [{"data_url": "data:image/png;base64,not-valid-base64!!!"}],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["reason"] == "decode"
@pytest.mark.asyncio
async def test_message_rejected_when_media_item_shape_wrong(tmp_path) -> None:
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "huh",
# Not a dict — plain string at the top level.
"media": ["data:image/png;base64,XXXX"],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["reason"] == "malformed"
@pytest.mark.asyncio
async def test_message_rejected_when_media_field_is_not_list() -> None:
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "huh",
"media": "not-a-list",
}
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["detail"] == "image_rejected"
assert err["reason"] == "malformed"
@pytest.mark.asyncio
async def test_failed_media_does_not_partially_persist(tmp_path) -> None:
"""If the second image is invalid, the first must not be forwarded."""
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": "mixed",
"media": [
{"data_url": _tiny_png_data_url()},
{"data_url": _data_url("application/pdf", b"%PDF-1.4")},
],
}
with patch(
"nanobot.channels.websocket.get_media_dir", return_value=tmp_path
):
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
# The first image was saved to disk (we don't roll it back — the caller
# is expected to not reference it) but the agent never sees the paths.
# That's the important invariant: no partial publish.
err = json.loads(mock_conn.send.call_args[0][0])
assert err["reason"] == "mime"
@pytest.mark.asyncio
async def test_rejects_empty_text_without_media() -> None:
"""When no media is attached, whitespace-only content is still rejected
(matches the existing behavior for backward compat)."""
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": " ",
}
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["detail"] == "missing content"
@pytest.mark.asyncio
async def test_non_string_content_still_rejected() -> None:
channel = _make_channel()
mock_conn = AsyncMock()
envelope = {
"type": "message",
"chat_id": "abc123",
"content": 42,
}
await channel._dispatch_envelope(mock_conn, "client-1", envelope)
channel._handle_message.assert_not_awaited()
err = json.loads(mock_conn.send.call_args[0][0])
assert err["detail"] == "missing content"
@@ -0,0 +1,375 @@
"""Tests for the signed ``/api/media/<sig>/<payload>`` route and its replay
integration on ``/api/sessions/<key>/messages``.
The route is the return path for images attached to persisted user turns:
:meth:`WebSocketChannel._sign_media_path` mints URLs during session reads,
and :meth:`WebSocketChannel._handle_media_fetch` serves the bytes back.
These tests cover the two halves end-to-end plus the adversarial edges
(bad signatures, ``..`` traversal, non-existent files, non-image types).
"""
from __future__ import annotations
import asyncio
import functools
import hashlib
import hmac
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from nanobot.channels.websocket import (
WebSocketChannel,
_b64url_decode,
_b64url_encode,
)
from nanobot.session.manager import Session, SessionManager
# PNG magic bytes + a couple of sentinel bytes so we can verify byte-for-byte
# round-trip of the served payload. Stays under mimetype + size limits.
_PNG_BYTES = (
b"\x89PNG\r\n\x1a\n"
b"\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01"
b"\x08\x06\x00\x00\x00\x1f\x15\xc4\x89"
b"\x00\x00\x00\nIDATx\x9cc\x00\x00\x00\x02\x00\x01"
b"\x00\x00\x00\x00IEND\xaeB`\x82"
)
def _ch(
bus: Any,
*,
session_manager: SessionManager | None = None,
port: int,
) -> WebSocketChannel:
return WebSocketChannel(
{
"enabled": True,
"allowFrom": ["*"],
"host": "127.0.0.1",
"port": port,
"path": "/",
"websocketRequiresToken": False,
},
bus,
session_manager=session_manager,
)
@pytest.fixture()
def bus() -> MagicMock:
b = MagicMock()
b.publish_inbound = AsyncMock()
return b
async def _http_get(
url: str, headers: dict[str, str] | None = None
) -> httpx.Response:
return await asyncio.to_thread(
functools.partial(httpx.get, url, headers=headers or {}, timeout=5.0)
)
# ---------------------------------------------------------------------------
# _sign_media_path: the URL minter
# ---------------------------------------------------------------------------
def test_sign_media_path_rejects_paths_outside_media_root(
bus: MagicMock, tmp_path: Path
) -> None:
"""Paths that resolve outside ``get_media_dir()`` must not be signed.
This is the single most important invariant of the whole scheme:
if the minter ever signed an arbitrary path, the HMAC would legitimise
it for the fetch handler and we'd hand out a disk-read primitive.
"""
outside = tmp_path / "secrets" / "cred.txt"
outside.parent.mkdir()
outside.write_text("nope")
media = tmp_path / "media"
media.mkdir()
channel = _ch(bus, port=0)
with patch("nanobot.channels.websocket.get_media_dir", return_value=media):
assert channel._sign_media_path(outside) is None
# Traversal via the media root is also rejected — the resolve() step
# normalises ``..`` out before the relative_to check.
assert channel._sign_media_path(media / ".." / "secrets" / "cred.txt") is None
def test_sign_media_path_round_trips_via_hmac(
bus: MagicMock, tmp_path: Path
) -> None:
"""The signature embeds exactly ``HMAC-SHA256(secret, payload)[:16]``."""
media = tmp_path / "media"
media.mkdir()
(media / "a.png").write_bytes(_PNG_BYTES)
channel = _ch(bus, port=0)
with patch("nanobot.channels.websocket.get_media_dir", return_value=media):
url = channel._sign_media_path(media / "a.png")
assert url is not None
assert url.startswith("/api/media/")
sig, payload = url[len("/api/media/"):].split("/", 1)
expected = hmac.new(
channel._media_secret, payload.encode("ascii"), hashlib.sha256
).digest()[:16]
assert _b64url_decode(sig) == expected
# The payload decodes back to the *relative* path — no absolute-path leaks.
assert _b64url_decode(payload).decode() == "a.png"
# ---------------------------------------------------------------------------
# /api/media/<sig>/<payload>: the serving handler
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_media_route_serves_signed_file(
bus: MagicMock, tmp_path: Path
) -> None:
"""Valid signature + existing file => 200 with correct bytes + MIME."""
media = tmp_path / "media"
media.mkdir()
target = media / "round-trip.png"
target.write_bytes(_PNG_BYTES)
channel = _ch(bus, port=29920)
with patch("nanobot.channels.websocket.get_media_dir", return_value=media):
url_path = channel._sign_media_path(target)
assert url_path is not None
server_task = asyncio.create_task(channel.start())
await asyncio.sleep(0.3)
try:
resp = await _http_get(f"http://127.0.0.1:29920{url_path}")
finally:
await channel.stop()
await server_task
assert resp.status_code == 200
assert resp.content == _PNG_BYTES
assert resp.headers["content-type"].startswith("image/png")
# Immutable cache header lets the browser skip round-trips on replay.
assert "immutable" in resp.headers.get("cache-control", "")
@pytest.mark.asyncio
async def test_media_route_rejects_bad_signature(
bus: MagicMock, tmp_path: Path
) -> None:
"""A payload re-signed with a different secret must 401.
Protects against a restart: old URLs baked into a stale tab become
un-forgeable once ``_media_secret`` regenerates.
"""
media = tmp_path / "media"
media.mkdir()
(media / "f.png").write_bytes(_PNG_BYTES)
channel = _ch(bus, port=29921)
with patch("nanobot.channels.websocket.get_media_dir", return_value=media):
good = channel._sign_media_path(media / "f.png")
assert good is not None
_, payload = good[len("/api/media/"):].split("/", 1)
# Forge a sig with a *different* secret.
forged_mac = hmac.new(
b"\x00" * 32, payload.encode("ascii"), hashlib.sha256
).digest()[:16]
forged = f"/api/media/{_b64url_encode(forged_mac)}/{payload}"
server_task = asyncio.create_task(channel.start())
await asyncio.sleep(0.3)
try:
resp = await _http_get(f"http://127.0.0.1:29921{forged}")
finally:
await channel.stop()
await server_task
assert resp.status_code == 401
@pytest.mark.asyncio
async def test_media_route_rejects_path_traversal_payload(
bus: MagicMock, tmp_path: Path
) -> None:
"""Even a validly-signed ``..`` payload must not escape the media root.
The signer never *emits* such payloads, but an attacker who somehow
obtained the secret (or the channel was misconfigured) must still be
stopped by the resolve()+relative_to() guard in the serving path.
"""
media = tmp_path / "media"
media.mkdir()
secret_file = tmp_path / "secret.txt"
secret_file.write_text("classified")
channel = _ch(bus, port=29922)
# Hand-craft a traversal payload the legit signer would refuse to mint.
payload = _b64url_encode(b"../secret.txt")
mac = hmac.new(
channel._media_secret, payload.encode("ascii"), hashlib.sha256
).digest()[:16]
url = f"/api/media/{_b64url_encode(mac)}/{payload}"
with patch("nanobot.channels.websocket.get_media_dir", return_value=media):
server_task = asyncio.create_task(channel.start())
await asyncio.sleep(0.3)
try:
resp = await _http_get(f"http://127.0.0.1:29922{url}")
finally:
await channel.stop()
await server_task
assert resp.status_code == 404
assert b"classified" not in resp.content
@pytest.mark.asyncio
async def test_media_route_404s_missing_file(
bus: MagicMock, tmp_path: Path
) -> None:
"""A signed URL for a file that no longer exists degrades to 404 so the
client can fall back to the placeholder tile instead of breaking."""
media = tmp_path / "media"
media.mkdir()
target = media / "gone.png"
target.write_bytes(_PNG_BYTES)
channel = _ch(bus, port=29923)
with patch("nanobot.channels.websocket.get_media_dir", return_value=media):
url_path = channel._sign_media_path(target)
assert url_path is not None
target.unlink() # the file vanishes between signing and fetching
server_task = asyncio.create_task(channel.start())
await asyncio.sleep(0.3)
try:
resp = await _http_get(f"http://127.0.0.1:29923{url_path}")
finally:
await channel.stop()
await server_task
assert resp.status_code == 404
@pytest.mark.asyncio
async def test_media_route_degrades_non_image_to_octet_stream(
bus: MagicMock, tmp_path: Path
) -> None:
"""A non-image extension must not be served as its native MIME.
Defence-in-depth: if media_dir ever contained (say) an HTML file, we
do not want the browser to render it as HTML via the signed route.
"""
media = tmp_path / "media"
media.mkdir()
(media / "scary.html").write_bytes(b"<script>alert(1)</script>")
channel = _ch(bus, port=29924)
with patch("nanobot.channels.websocket.get_media_dir", return_value=media):
payload = _b64url_encode(b"scary.html")
mac = hmac.new(
channel._media_secret, payload.encode("ascii"), hashlib.sha256
).digest()[:16]
url = f"/api/media/{_b64url_encode(mac)}/{payload}"
server_task = asyncio.create_task(channel.start())
await asyncio.sleep(0.3)
try:
resp = await _http_get(f"http://127.0.0.1:29924{url}")
finally:
await channel.stop()
await server_task
assert resp.status_code == 200
assert resp.headers["content-type"].startswith("application/octet-stream")
# ---------------------------------------------------------------------------
# /api/sessions/<key>/messages: media_urls hydration on session read
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_session_messages_exposes_signed_media_urls(
bus: MagicMock, tmp_path: Path
) -> None:
"""The read path must map persisted ``media`` paths onto signed URLs
and strip the raw path — the client never learns the server's layout."""
media = tmp_path / "media"
media.mkdir()
img = media / "u.png"
img.write_bytes(_PNG_BYTES)
sm = SessionManager(tmp_path / "ws_state")
sess = Session(key="websocket:media-hydrate")
sess.add_message("user", "look at this", media=[str(img)])
sess.add_message("assistant", "nice")
sm.save(sess)
channel = _ch(bus, session_manager=sm, port=29925)
with patch("nanobot.channels.websocket.get_media_dir", return_value=media):
server_task = asyncio.create_task(channel.start())
await asyncio.sleep(0.3)
try:
boot = await _http_get("http://127.0.0.1:29925/webui/bootstrap")
token = boot.json()["token"]
auth = {"Authorization": f"Bearer {token}"}
resp = await _http_get(
"http://127.0.0.1:29925/api/sessions/websocket:media-hydrate/messages",
headers=auth,
)
body = resp.json()
# The signed URL round-trips end-to-end: fetching it yields the same bytes.
user_msg = next(m for m in body["messages"] if m["role"] == "user")
urls = user_msg["media_urls"]
assert isinstance(urls, list) and len(urls) == 1
assert urls[0]["name"] == "u.png"
assert urls[0]["url"].startswith("/api/media/")
# Raw paths must not leak to the wire.
assert "media" not in user_msg
# And the URL actually works.
fetched = await _http_get(f"http://127.0.0.1:29925{urls[0]['url']}")
assert fetched.status_code == 200
assert fetched.content == _PNG_BYTES
finally:
await channel.stop()
await server_task
@pytest.mark.asyncio
async def test_session_messages_skips_vanished_media(
bus: MagicMock, tmp_path: Path
) -> None:
"""Paths that no longer resolve inside the media root produce no URL —
the message is still delivered, just without the preview."""
media = tmp_path / "media"
media.mkdir()
sm = SessionManager(tmp_path / "ws_state")
sess = Session(key="websocket:vanished")
sess.add_message("user", "missing pic", media=[str(media / "absent.png")])
sm.save(sess)
channel = _ch(bus, session_manager=sm, port=29926)
with patch("nanobot.channels.websocket.get_media_dir", return_value=media):
server_task = asyncio.create_task(channel.start())
await asyncio.sleep(0.3)
try:
boot = await _http_get("http://127.0.0.1:29926/webui/bootstrap")
token = boot.json()["token"]
resp = await _http_get(
"http://127.0.0.1:29926/api/sessions/websocket:vanished/messages",
headers={"Authorization": f"Bearer {token}"},
)
user_msg = next(m for m in resp.json()["messages"] if m["role"] == "user")
# absent.png lives inside the media root so it *does* get a signed
# URL (we don't stat the file at signing time — that would slow
# the listing). Fetching the URL is where the 404 surfaces.
urls = user_msg.get("media_urls") or []
assert len(urls) == 1
fetched = await _http_get(f"http://127.0.0.1:29926{urls[0]['url']}")
assert fetched.status_code == 404
assert "media" not in user_msg
finally:
await channel.stop()
await server_task
+75
View File
@@ -0,0 +1,75 @@
"""Tests for ``nanobot.utils.media_decode``."""
from __future__ import annotations
import base64
import pytest
from nanobot.utils.media_decode import (
DEFAULT_MAX_BYTES,
FileSizeExceeded,
MAX_FILE_SIZE,
save_base64_data_url,
)
def _data_url(payload: bytes, mime: str = "image/png") -> str:
return f"data:{mime};base64,{base64.b64encode(payload).decode()}"
def test_saves_png_with_correct_extension(tmp_path) -> None:
result = save_base64_data_url(_data_url(b"fake png"), tmp_path)
assert result is not None
assert result.endswith(".png")
assert (tmp_path / result.split("/")[-1]).read_bytes() == b"fake png"
def test_returns_none_for_malformed_data_url(tmp_path) -> None:
assert save_base64_data_url("not-a-data-url", tmp_path) is None
def test_returns_none_for_broken_base64(tmp_path) -> None:
# Python's b64decode strips non-alphabet chars by default, so we need a
# payload whose alphabet-filtered length breaks padding.
assert save_base64_data_url("data:image/png;base64,not-valid-base64!!!", tmp_path) is None
def test_unknown_mime_falls_back_to_bin(tmp_path) -> None:
result = save_base64_data_url(_data_url(b"xyz", mime="unknown/type"), tmp_path)
assert result is not None
assert result.endswith(".bin")
def test_default_limit_is_10mb(tmp_path) -> None:
"""Backwards-compatible default — the API path depends on this."""
assert DEFAULT_MAX_BYTES == 10 * 1024 * 1024
assert MAX_FILE_SIZE == 10 * 1024 * 1024
oversized = b"x" * (11 * 1024 * 1024)
with pytest.raises(FileSizeExceeded, match="10MB limit"):
save_base64_data_url(_data_url(oversized), tmp_path)
def test_explicit_max_bytes_overrides_default(tmp_path) -> None:
"""WS channel passes 8 MB; a 9 MB payload should be rejected there even
though it would pass the 10 MB API limit."""
payload = b"y" * (9 * 1024 * 1024)
with pytest.raises(FileSizeExceeded, match="8MB limit"):
save_base64_data_url(_data_url(payload), tmp_path, max_bytes=8 * 1024 * 1024)
def test_saved_file_lives_under_media_dir(tmp_path) -> None:
result = save_base64_data_url(_data_url(b"ok"), tmp_path)
assert result is not None
assert result.startswith(str(tmp_path))
def test_legacy_symbols_reexported_from_api_server() -> None:
"""Existing tests import ``_save_base64_data_url`` / ``_FileSizeExceeded``
from ``nanobot.api.server`` — keep the aliases working."""
from nanobot.api import server
assert server._save_base64_data_url is save_base64_data_url
assert server._FileSizeExceeded is FileSizeExceeded
assert server.MAX_FILE_SIZE == MAX_FILE_SIZE
+199
View File
@@ -0,0 +1,199 @@
import { useCallback, useEffect, useMemo } from "react";
import * as DialogPrimitive from "@radix-ui/react-dialog";
import { ChevronLeft, ChevronRight, X } from "lucide-react";
import { useTranslation } from "react-i18next";
import { cn } from "@/lib/utils";
import type { UIImage } from "@/lib/types";
interface ImageLightboxProps {
images: UIImage[];
index: number | null;
onIndexChange: (index: number) => void;
onOpenChange: (open: boolean) => void;
}
/**
* Modal image viewer. Uses the Radix Dialog primitives directly so we can
* fill the viewport (the shared `DialogContent` wrapper caps at max-w-lg,
* which is much too small for a photo preview).
*
* Implementation notes:
* - `translate3d` + `will-change: transform` promote the image to a GPU
* compositing layer so open/swap stays at 60 FPS on long threads.
* - Adjacent images are rendered in hidden `<img>` tags so the browser
* decodes them eagerly; pressing left/right feels instant.
* - Radix handles `Escape` + focus trapping; we only wire up ←/→ + Home/End.
* - Respects `prefers-reduced-motion` by dropping the fade + zoom-in
* keyframes via `motion-reduce:*` variants.
*/
export function ImageLightbox({
images,
index,
onIndexChange,
onOpenChange,
}: ImageLightboxProps) {
const { t } = useTranslation();
const open = index !== null;
const total = images.length;
const current = index !== null ? images[index] : null;
const go = useCallback(
(delta: number) => {
if (index === null || total <= 1) return;
const next = (index + delta + total) % total;
onIndexChange(next);
},
[index, onIndexChange, total],
);
useEffect(() => {
if (!open) return;
const onKey = (e: KeyboardEvent) => {
if (e.key === "ArrowLeft") {
e.preventDefault();
go(-1);
} else if (e.key === "ArrowRight") {
e.preventDefault();
go(1);
} else if (e.key === "Home") {
e.preventDefault();
onIndexChange(0);
} else if (e.key === "End") {
e.preventDefault();
onIndexChange(total - 1);
}
};
window.addEventListener("keydown", onKey);
return () => window.removeEventListener("keydown", onKey);
}, [go, onIndexChange, open, total]);
// Neighbours we want the browser to decode eagerly.
const preload = useMemo(() => {
if (index === null || total <= 1) return [] as UIImage[];
const prev = images[(index - 1 + total) % total];
const next = images[(index + 1) % total];
return [prev, next].filter((i) => i && i.url);
}, [images, index, total]);
if (!current || !current.url) return null;
const hasMany = total > 1;
const counter = hasMany ? `${index! + 1} / ${total}` : null;
return (
<DialogPrimitive.Root open={open} onOpenChange={onOpenChange}>
<DialogPrimitive.Portal>
<DialogPrimitive.Overlay
className={cn(
"fixed inset-0 z-50 bg-black/80 backdrop-blur-sm",
"data-[state=open]:animate-in data-[state=closed]:animate-out",
"data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0",
"motion-reduce:data-[state=open]:animate-none motion-reduce:data-[state=closed]:animate-none",
)}
/>
<DialogPrimitive.Content
aria-label={current.name ?? t("lightbox.title")}
className={cn(
"fixed inset-0 z-50 flex items-center justify-center",
"focus:outline-none",
"data-[state=open]:animate-in data-[state=closed]:animate-out",
"data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0",
"data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95",
"motion-reduce:data-[state=open]:animate-none motion-reduce:data-[state=closed]:animate-none",
)}
>
<DialogPrimitive.Title className="sr-only">
{current.name ?? t("lightbox.title")}
</DialogPrimitive.Title>
<div
className="relative flex max-h-[92vh] max-w-[94vw] items-center justify-center"
style={{
transform: "translate3d(0,0,0)",
willChange: "transform",
}}
>
<img
key={current.url}
src={current.url}
alt={current.name ?? ""}
decoding="async"
draggable={false}
className="max-h-[92vh] max-w-[94vw] select-none rounded-[6px] object-contain shadow-2xl"
/>
</div>
{hasMany ? (
<>
<NavButton
side="left"
label={t("lightbox.prev")}
onClick={(e) => {
e.stopPropagation();
go(-1);
}}
/>
<NavButton
side="right"
label={t("lightbox.next")}
onClick={(e) => {
e.stopPropagation();
go(1);
}}
/>
<div className="pointer-events-none absolute bottom-5 left-1/2 -translate-x-1/2 rounded-full bg-black/55 px-3 py-1 text-xs font-medium text-white/90 tabular-nums">
{counter}
</div>
</>
) : null}
<DialogPrimitive.Close
aria-label={t("lightbox.close")}
className={cn(
"absolute right-4 top-4 grid h-9 w-9 place-items-center rounded-full",
"bg-black/55 text-white/90 hover:bg-black/70 hover:text-white",
"focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/70",
"transition-colors motion-reduce:transition-none",
)}
>
<X className="h-4 w-4" aria-hidden />
</DialogPrimitive.Close>
{/* Invisible preload — browser decodes adjacent images so prev/next swap is instant. */}
<div aria-hidden className="hidden">
{preload.map((img, i) => (
<img key={`${img.url}-${i}`} src={img.url} alt="" />
))}
</div>
</DialogPrimitive.Content>
</DialogPrimitive.Portal>
</DialogPrimitive.Root>
);
}
interface NavButtonProps {
side: "left" | "right";
label: string;
onClick: React.MouseEventHandler<HTMLButtonElement>;
}
function NavButton({ side, label, onClick }: NavButtonProps) {
const Icon = side === "left" ? ChevronLeft : ChevronRight;
return (
<button
type="button"
onClick={onClick}
aria-label={label}
className={cn(
"absolute top-1/2 -translate-y-1/2 grid h-11 w-11 place-items-center rounded-full",
"bg-black/55 text-white/90 hover:bg-black/70 hover:text-white",
"focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/70",
"transition-colors motion-reduce:transition-none",
side === "left" ? "left-4" : "right-4",
)}
>
<Icon className="h-5 w-5" aria-hidden />
</button>
);
}
+134 -12
View File
@@ -1,10 +1,11 @@
import { useState } from "react";
import { ChevronRight, Wrench } from "lucide-react";
import { ChevronRight, ImageIcon, Wrench } from "lucide-react";
import { useTranslation } from "react-i18next";
import { ImageLightbox } from "@/components/ImageLightbox";
import { MarkdownText } from "@/components/MarkdownText";
import { cn } from "@/lib/utils";
import type { UIMessage } from "@/lib/types";
import type { UIImage, UIMessage } from "@/lib/types";
interface MessageBubbleProps {
message: UIMessage;
@@ -27,22 +28,28 @@ export function MessageBubble({ message }: MessageBubbleProps) {
}
if (message.role === "user") {
const images = message.images ?? [];
const hasImages = images.length > 0;
const hasText = message.content.trim().length > 0;
return (
<div
className={cn(
"group ml-auto flex max-w-[min(85%,36rem)] items-center gap-2",
"group ml-auto flex max-w-[min(85%,36rem)] flex-col items-end gap-1.5",
baseAnim,
)}
>
<p
className={cn(
"ml-auto w-fit rounded-[18px] border border-border/60 bg-secondary/70 px-4 py-2",
"text-right text-[18px]/[1.8] whitespace-pre-wrap break-words",
"shadow-[0_10px_24px_-18px_rgba(0,0,0,0.55)]",
)}
>
{message.content}
</p>
{hasImages ? <UserImages images={images} /> : null}
{hasText ? (
<p
className={cn(
"ml-auto w-fit rounded-[18px] border border-border/60 bg-secondary/70 px-4 py-2",
"text-right text-[18px]/[1.8] whitespace-pre-wrap break-words",
"shadow-[0_10px_24px_-18px_rgba(0,0,0,0.55)]",
)}
>
{message.content}
</p>
) : null}
</div>
);
}
@@ -62,6 +69,121 @@ export function MessageBubble({ message }: MessageBubbleProps) {
);
}
/**
* Right-aligned preview row for images attached to a user turn.
*
* Visual follows agent-chat-ui: a single wrapping row of fixed-size square
* thumbnails that stay modest next to the text pill regardless of how many
* images are attached.
*
* The URL is expected to be a self-contained ``data:`` URL (the Composer
* hands the normalized base64 payload to the optimistic bubble so that the
* preview survives React StrictMode double-mount — blob URLs would be
* revoked by the Composer's cleanup before remount). Historical replays
* have no URL (the backend strips data URLs before persisting), so we
* render a labelled placeholder tile instead of a broken ``<img>``.
*/
function UserImages({ images }: { images: UIImage[] }) {
const { t } = useTranslation();
// Only real-URL images can open in the lightbox; historical-replay
// placeholders (no URL) have nothing to zoom into.
const viewable = images
.map((img, i) => ({ img, i }))
.filter(({ img }) => typeof img.url === "string" && img.url.length > 0);
const viewableImages = viewable.map(({ img }) => img);
const originalToViewable = new Map<number, number>(
viewable.map(({ i }, v) => [i, v]),
);
const [lightboxIndex, setLightboxIndex] = useState<number | null>(null);
return (
<>
<div className="ml-auto flex flex-wrap items-end justify-end gap-2">
{images.map((img, i) => (
<UserImageCell
key={`${img.url ?? "placeholder"}-${i}`}
image={img}
placeholderLabel={t("message.imageAttachment")}
openLabel={t("lightbox.open")}
onOpen={
originalToViewable.has(i)
? () => setLightboxIndex(originalToViewable.get(i)!)
: undefined
}
/>
))}
</div>
<ImageLightbox
images={viewableImages}
index={lightboxIndex}
onIndexChange={setLightboxIndex}
onOpenChange={(open) => {
if (!open) setLightboxIndex(null);
}}
/>
</>
);
}
function UserImageCell({
image,
placeholderLabel,
openLabel,
onOpen,
}: {
image: UIImage;
placeholderLabel: string;
openLabel: string;
onOpen?: () => void;
}) {
const hasUrl = typeof image.url === "string" && image.url.length > 0;
const tileClasses = cn(
"relative h-24 w-24 overflow-hidden rounded-[14px] border border-border/60 bg-muted/40",
"shadow-[0_6px_18px_-14px_rgba(0,0,0,0.45)]",
);
if (hasUrl && onOpen) {
return (
<button
type="button"
onClick={onOpen}
aria-label={image.name ? `${openLabel}: ${image.name}` : openLabel}
title={image.name ?? undefined}
className={cn(
tileClasses,
"cursor-zoom-in transition-transform duration-150 motion-reduce:transition-none",
"hover:scale-[1.02] hover:ring-2 hover:ring-primary/30",
"focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-primary/50",
)}
>
<img
src={image.url}
alt={image.name ?? ""}
loading="lazy"
decoding="async"
draggable={false}
className="h-full w-full object-cover"
/>
</button>
);
}
return (
<div className={tileClasses} title={image.name ?? undefined}>
<div
className="flex h-full w-full flex-col items-center justify-center gap-1 px-2 text-[11px] text-muted-foreground"
aria-label={placeholderLabel}
>
<ImageIcon className="h-4 w-4 flex-none" aria-hidden />
<span className="line-clamp-2 text-center leading-tight">
{image.name ?? placeholderLabel}
</span>
</div>
</div>
);
}
/** Blinking cursor appended at the end of streaming text. */
function StreamCursor() {
const { t } = useTranslation();
@@ -0,0 +1,72 @@
import { AlertTriangle, X } from "lucide-react";
import { useTranslation } from "react-i18next";
import { Button } from "@/components/ui/button";
import { cn } from "@/lib/utils";
import type { StreamError } from "@/lib/nanobot-client";
interface StreamErrorNoticeProps {
error: StreamError;
onDismiss: () => void;
}
/**
* Dismissible banner that surfaces transport-level faults the user needs to
* know about. Rendered above the composer so the message the fault referred
* to remains in view just above. ``role="alert"`` + ``aria-live="assertive"``
* ensures screen readers announce the failure.
*/
export function StreamErrorNotice({ error, onDismiss }: StreamErrorNoticeProps) {
const { t } = useTranslation();
const { title, body } = resolveCopy(error, t);
return (
<div
role="alert"
aria-live="assertive"
className={cn(
"mb-2 flex items-start gap-2 rounded-lg border border-destructive/30",
"bg-destructive/10 px-3 py-2 text-[12px] leading-5 text-destructive",
"animate-in fade-in-0 slide-in-from-bottom-1",
)}
>
<AlertTriangle
className="mt-0.5 h-4 w-4 shrink-0"
aria-hidden
/>
<div className="flex-1">
<p className="font-medium">{title}</p>
<p className="mt-0.5 text-destructive/80">{body}</p>
</div>
<Button
variant="ghost"
size="icon"
onClick={onDismiss}
aria-label={t("common.dismiss")}
className="h-6 w-6 shrink-0 text-destructive hover:bg-destructive/15 hover:text-destructive"
>
<X className="h-3.5 w-3.5" />
</Button>
</div>
);
}
function resolveCopy(
error: StreamError,
t: (key: string) => string,
): { title: string; body: string } {
switch (error.kind) {
case "message_too_big":
return {
title: t("errors.messageTooBig.title"),
body: t("errors.messageTooBig.body"),
};
default: {
// Exhaustiveness guard: if a new StreamError kind is added, TS will
// complain here until we add a corresponding i18n branch.
const _exhaustive: never = error.kind;
return { title: String(_exhaustive), body: "" };
}
}
}
+305 -9
View File
@@ -1,12 +1,43 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { ArrowUp } from "lucide-react";
import {
useCallback,
useEffect,
useMemo,
useRef,
useState,
type KeyboardEvent as ReactKeyboardEvent,
} from "react";
import {
ArrowUp,
ImageIcon,
Loader2,
Paperclip,
X,
} from "lucide-react";
import { useTranslation } from "react-i18next";
import { Button } from "@/components/ui/button";
import {
useAttachedImages,
type AttachedImage,
type AttachmentError,
MAX_IMAGES_PER_MESSAGE,
} from "@/hooks/useAttachedImages";
import { useClipboardAndDrop } from "@/hooks/useClipboardAndDrop";
import type { SendImage } from "@/hooks/useNanobotStream";
import { cn } from "@/lib/utils";
/** ``<input accept>``: aligned with the server's MIME whitelist. SVG is
* deliberately excluded to avoid an embedded-script XSS surface. */
const ACCEPT_ATTR = "image/png,image/jpeg,image/webp,image/gif";
function formatBytes(n: number): string {
if (n < 1024) return `${n} B`;
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
}
interface ThreadComposerProps {
onSend: (content: string) => void;
onSend: (content: string, images?: SendImage[]) => void;
disabled?: boolean;
placeholder?: string;
modelLabel?: string | null;
@@ -22,11 +53,47 @@ export function ThreadComposer({
}: ThreadComposerProps) {
const { t } = useTranslation();
const [value, setValue] = useState("");
const [inlineError, setInlineError] = useState<string | null>(null);
const textareaRef = useRef<HTMLTextAreaElement>(null);
const fileInputRef = useRef<HTMLInputElement>(null);
const chipRefs = useRef(new Map<string, HTMLButtonElement>());
const isHero = variant === "hero";
const resolvedPlaceholder =
placeholder ?? t("thread.composer.placeholderThread");
const { images, enqueue, remove, clear, encoding, full } =
useAttachedImages();
const formatRejection = useCallback(
(reason: AttachmentError): string => {
const key = `thread.composer.imageRejected.${reason}`;
return t(key, { max: MAX_IMAGES_PER_MESSAGE });
},
[t],
);
const addFiles = useCallback(
(files: File[]) => {
if (files.length === 0) return;
const { rejected } = enqueue(files);
if (rejected.length > 0) {
setInlineError(formatRejection(rejected[0].reason));
} else {
setInlineError(null);
}
},
[enqueue, formatRejection],
);
const {
isDragging,
onPaste,
onDragEnter,
onDragOver,
onDragLeave,
onDrop,
} = useClipboardAndDrop(addFiles);
useEffect(() => {
if (disabled) return;
const el = textareaRef.current;
@@ -35,11 +102,43 @@ export function ThreadComposer({
return () => cancelAnimationFrame(id);
}, [disabled]);
const readyImages = useMemo(
() => images.filter((img): img is AttachedImage & { dataUrl: string } =>
img.status === "ready" && typeof img.dataUrl === "string",
),
[images],
);
const hasErrors = images.some((img) => img.status === "error");
const canSend =
!disabled
&& !encoding
&& !hasErrors
&& (value.trim().length > 0 || readyImages.length > 0);
const submit = useCallback(() => {
if (!canSend) return;
const trimmed = value.trim();
if (!trimmed || disabled) return;
onSend(trimmed);
// Share the same normalized ``data:`` URL with both the wire payload and
// the optimistic bubble preview: data URLs are self-contained (no blob
// lifetime, safe under React StrictMode double-mount) and keep the
// bubble in sync with whatever the backend actually sees.
const payload: SendImage[] | undefined =
readyImages.length > 0
? readyImages.map((img) => ({
media: {
data_url: img.dataUrl,
name: img.file.name,
},
preview: { url: img.dataUrl, name: img.file.name },
}))
: undefined;
onSend(trimmed, payload);
setValue("");
setInlineError(null);
// Bubble owns the data URL copy; safe to revoke every staged blob
// preview here without affecting the rendered message.
clear();
requestAnimationFrame(() => {
const el = textareaRef.current;
if (el) {
@@ -47,9 +146,9 @@ export function ThreadComposer({
el.focus();
}
});
}, [disabled, onSend, value]);
}, [canSend, clear, onSend, readyImages, value]);
const onKeyDown: React.KeyboardEventHandler<HTMLTextAreaElement> = (e) => {
const onKeyDown = (e: ReactKeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Enter" && !e.shiftKey && !e.nativeEvent.isComposing) {
e.preventDefault();
submit();
@@ -62,12 +161,55 @@ export function ThreadComposer({
el.style.height = `${Math.min(el.scrollHeight, 260)}px`;
};
const onFilePick: React.ChangeEventHandler<HTMLInputElement> = (e) => {
const files = Array.from(e.target.files ?? []);
e.target.value = "";
addFiles(files);
};
const removeChip = useCallback(
(id: string) => {
const { nextFocusId } = remove(id);
setInlineError(null);
requestAnimationFrame(() => {
const el = nextFocusId ? chipRefs.current.get(nextFocusId) : null;
if (el) {
el.focus();
} else {
textareaRef.current?.focus();
}
});
},
[remove],
);
const onChipKey = useCallback(
(id: string) => (e: ReactKeyboardEvent<HTMLButtonElement>) => {
if (
e.key === "Delete" ||
e.key === "Backspace" ||
e.key === "Enter" ||
e.key === " "
) {
e.preventDefault();
removeChip(id);
}
},
[removeChip],
);
const attachButtonDisabled = disabled || full;
return (
<form
onSubmit={(e) => {
e.preventDefault();
submit();
}}
onDragEnter={onDragEnter}
onDragOver={onDragOver}
onDragLeave={onDragLeave}
onDrop={onDrop}
className={cn("w-full", isHero ? "px-0" : "px-1 pb-1.5 pt-1 sm:px-0")}
>
<div
@@ -78,14 +220,44 @@ export function ThreadComposer({
: "max-w-[49.5rem] rounded-[16px] border border-border/70 bg-card/55",
"focus-within:bg-card/70 focus-within:ring-1 focus-within:ring-foreground/8",
disabled && "opacity-60",
isDragging && "ring-2 ring-primary/40 motion-reduce:ring-0 motion-reduce:border-primary",
)}
>
{images.length > 0 ? (
<div
className="flex flex-wrap gap-2 px-3 pt-3"
aria-label={t("thread.composer.attachImage")}
>
{images.map((img) => (
<AttachmentChip
key={img.id}
image={img}
labelRemove={t("thread.composer.remove")}
labelEncoding={t("thread.composer.encoding")}
normalizedHint={(orig, current) =>
t("thread.composer.normalizedSizeHint", {
orig: formatBytes(orig),
current: formatBytes(current),
})
}
formatError={formatRejection}
onRemove={() => removeChip(img.id)}
onKeyDown={onChipKey(img.id)}
registerRef={(el) => {
if (el) chipRefs.current.set(img.id, el);
else chipRefs.current.delete(img.id);
}}
/>
))}
</div>
) : null}
<textarea
ref={textareaRef}
value={value}
onChange={(e) => setValue(e.target.value)}
onInput={onInput}
onKeyDown={onKeyDown}
onPaste={onPaste}
rows={1}
placeholder={resolvedPlaceholder}
disabled={disabled}
@@ -100,6 +272,17 @@ export function ThreadComposer({
"disabled:cursor-not-allowed",
)}
/>
{inlineError ? (
<div
role="alert"
className={cn(
"mx-3 mb-1 rounded-md border border-destructive/40 bg-destructive/8 px-2.5 py-1",
"text-[11.5px] font-medium text-destructive",
)}
>
{inlineError}
</div>
) : null}
<div
className={cn(
"flex items-center justify-between gap-2",
@@ -107,6 +290,28 @@ export function ThreadComposer({
)}
>
<div className="flex min-w-0 items-center gap-2">
<input
ref={fileInputRef}
type="file"
accept={ACCEPT_ATTR}
multiple
hidden
onChange={onFilePick}
/>
<Button
type="button"
size="icon"
variant="ghost"
disabled={attachButtonDisabled}
aria-label={t("thread.composer.attachImage")}
onClick={() => fileInputRef.current?.click()}
className={cn(
"rounded-full text-muted-foreground hover:text-foreground",
isHero ? "h-8.5 w-8.5" : "h-7.5 w-7.5",
)}
>
<Paperclip className={cn(isHero ? "h-4 w-4" : "h-3.5 w-3.5")} />
</Button>
{modelLabel ? (
<span
title={modelLabel}
@@ -131,12 +336,12 @@ export function ThreadComposer({
<Button
type="submit"
size="icon"
disabled={disabled || !value.trim()}
disabled={!canSend}
aria-label={t("thread.composer.send")}
className={cn(
"rounded-full border border-border/70 bg-secondary/85 text-secondary-foreground shadow-none transition-transform hover:bg-accent",
isHero ? "h-8.5 w-8.5" : "h-7.5 w-7.5",
value.trim() && !disabled && "hover:scale-[1.03] active:scale-95",
canSend && "hover:scale-[1.03] active:scale-95",
)}
>
<ArrowUp className={cn(isHero ? "h-4.5 w-4.5" : "h-4 w-4")} />
@@ -146,3 +351,94 @@ export function ThreadComposer({
</form>
);
}
interface AttachmentChipProps {
image: AttachedImage;
labelRemove: string;
labelEncoding: string;
normalizedHint: (origBytes: number, currentBytes: number) => string;
formatError: (reason: AttachmentError) => string;
onRemove: () => void;
onKeyDown: (e: ReactKeyboardEvent<HTMLButtonElement>) => void;
registerRef: (el: HTMLButtonElement | null) => void;
}
function AttachmentChip({
image,
labelRemove,
labelEncoding,
normalizedHint,
formatError,
onRemove,
onKeyDown,
registerRef,
}: AttachmentChipProps) {
const sizeLabel =
image.status === "ready" && image.normalized && image.encodedBytes
? normalizedHint(image.file.size, image.encodedBytes)
: formatBytes(image.file.size);
const tone =
image.status === "error"
? "border-destructive/40 bg-destructive/5 text-destructive"
: "border-border/70 bg-muted/60";
return (
<div
className={cn(
"group relative flex items-center gap-2 rounded-[12px] border px-2 py-1.5",
"transition-colors motion-reduce:transition-none",
tone,
)}
data-testid="composer-chip"
>
<div className="relative h-10 w-10 overflow-hidden rounded-md bg-background">
{image.previewUrl ? (
<img
src={image.previewUrl}
alt=""
aria-hidden
loading="eager"
draggable={false}
className="h-full w-full object-cover"
/>
) : (
<div className="flex h-full w-full items-center justify-center">
<ImageIcon className="h-4 w-4 text-muted-foreground" aria-hidden />
</div>
)}
{image.status === "encoding" ? (
<div
className="absolute inset-0 flex items-center justify-center bg-background/60"
aria-label={labelEncoding}
>
<Loader2 className="h-4 w-4 animate-spin motion-reduce:animate-none" aria-hidden />
</div>
) : null}
</div>
<div className="flex min-w-0 flex-col text-[11.5px] leading-4">
<span className="truncate max-w-[14rem] font-medium" title={image.file.name}>
{image.file.name}
</span>
<span className="truncate text-muted-foreground">
{image.status === "error" && image.error
? formatError(image.error)
: sizeLabel}
</span>
</div>
<button
type="button"
ref={registerRef}
onClick={onRemove}
onKeyDown={onKeyDown}
aria-label={labelRemove}
className={cn(
"ml-1 grid h-5 w-5 flex-none place-items-center rounded-full",
"text-muted-foreground/80 hover:bg-foreground/8 hover:text-foreground",
"focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-foreground/30",
)}
>
<X className="h-3.5 w-3.5" aria-hidden />
</button>
</div>
);
}
+42 -29
View File
@@ -3,6 +3,7 @@ import { useTranslation } from "react-i18next";
import { ThreadComposer } from "@/components/thread/ThreadComposer";
import { ThreadHeader } from "@/components/thread/ThreadHeader";
import { StreamErrorNotice } from "@/components/thread/StreamErrorNotice";
import { ThreadViewport } from "@/components/thread/ThreadViewport";
import { useNanobotStream } from "@/hooks/useNanobotStream";
import { useSessionHistory } from "@/hooks/useSessions";
@@ -47,10 +48,14 @@ export function ThreadShell({
if (!chatId) return historical;
return messageCacheRef.current.get(chatId) ?? historical;
}, [chatId, historical]);
const { messages, isStreaming, send, setMessages } = useNanobotStream(
chatId,
initial,
);
const {
messages,
isStreaming,
send,
setMessages,
streamError,
dismissStreamError,
} = useNanobotStream(chatId, initial);
const showHeroComposer = messages.length === 0 && !loading;
useEffect(() => {
@@ -140,31 +145,39 @@ export function ThreadShell({
isStreaming={isStreaming}
emptyState={emptyState}
composer={
session ? (
<ThreadComposer
onSend={send}
disabled={!chatId}
placeholder={
showHeroComposer
? t("thread.composer.placeholderHero")
: t("thread.composer.placeholderThread")
}
modelLabel={toModelBadgeLabel(modelName)}
variant={showHeroComposer ? "hero" : "thread"}
/>
) : (
<ThreadComposer
onSend={handleWelcomeSend}
disabled={booting}
placeholder={
booting
? t("thread.composer.placeholderOpening")
: t("thread.composer.placeholderHero")
}
modelLabel={toModelBadgeLabel(modelName)}
variant="hero"
/>
)
<>
{streamError ? (
<StreamErrorNotice
error={streamError}
onDismiss={dismissStreamError}
/>
) : null}
{session ? (
<ThreadComposer
onSend={send}
disabled={!chatId}
placeholder={
showHeroComposer
? t("thread.composer.placeholderHero")
: t("thread.composer.placeholderThread")
}
modelLabel={toModelBadgeLabel(modelName)}
variant={showHeroComposer ? "hero" : "thread"}
/>
) : (
<ThreadComposer
onSend={handleWelcomeSend}
disabled={booting}
placeholder={
booting
? t("thread.composer.placeholderOpening")
: t("thread.composer.placeholderHero")
}
modelLabel={toModelBadgeLabel(modelName)}
variant="hero"
/>
)}
</>
}
/>
</section>
+233
View File
@@ -0,0 +1,233 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { encodeImage, type EncodeFailure } from "@/lib/imageEncode";
/** Lifecycle stages of one attachment:
*
* - ``encoding`` — posted to the Worker; chip shows a spinner
* - ``ready`` — ``dataUrl`` available; safe to submit
* - ``error`` — validation / decode failure; chip shows inline error
*/
export type AttachmentStatus = "encoding" | "ready" | "error";
export interface AttachedImage {
id: string;
file: File;
/** Optimistic ``blob:`` preview URL; revoked on ``remove`` / ``clear`` /
* unmount. */
previewUrl: string;
status: AttachmentStatus;
/** Populated when ``status === "ready"``. */
dataUrl?: string;
/** Size of the final encoded payload (base64 bytes decoded). */
encodedBytes?: number;
/** Whether the Worker re-encoded the image to hit the size budget. */
normalized?: boolean;
/** Human-readable validation / encoding error when ``status === "error"``. */
error?: AttachmentError;
}
/** Machine-readable rejection reasons surfaced as inline chip errors.
*
* Callers localize these via the ``composer.imageRejected.*`` i18n table. */
export type AttachmentError =
| "unsupported_type" // server whitelist excludes this MIME
| "too_many_images" // per-message cap (4) reached before enqueue
| "magic_mismatch" // extension lies about the real content
| "decode_failed" // Worker couldn't decode / re-encode
| "too_large" // even after normalization we exceed the budget
| "io"; // file read failed at the browser layer
export const MAX_IMAGES_PER_MESSAGE = 4;
/** MIME whitelist — mirrors the server's and the ``<input accept>`` attr. */
const ACCEPTED_MIMES: ReadonlySet<string> = new Set([
"image/png",
"image/jpeg",
"image/webp",
"image/gif",
]);
function uuid(): string {
if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
return (crypto as Crypto).randomUUID();
}
return `img-${Date.now()}-${Math.random().toString(36).slice(2)}`;
}
function mapEncodeFailure(reason: EncodeFailure["reason"]): AttachmentError {
switch (reason) {
case "invalid_mime":
case "magic_mismatch":
return "magic_mismatch";
case "too_large_after_normalize":
return "too_large";
case "io":
return "io";
case "decode_failed":
default:
return "decode_failed";
}
}
export interface UseAttachedImagesApi {
images: AttachedImage[];
/** Enqueue new files. Returns the list of rejected files so the caller can
* surface inline errors. Files rejected client-side (wrong MIME, limit) are
* *not* added to ``images`` — only recoverable encoding failures show up as
* error chips. */
enqueue: (files: Iterable<File>) => {
rejected: Array<{ file: File; reason: AttachmentError }>;
};
remove: (id: string) => { nextFocusId: string | null };
/** Revoke every staged blob URL and drop all attachments. Called after a
* successful submit — the optimistic bubble holds onto an independent
* ``data:`` URL so tearing down blob previews here is safe. */
clear: () => void;
/** ``true`` when at least one image is still encoding — Send should wait. */
encoding: boolean;
/** ``true`` when we've hit ``MAX_IMAGES_PER_MESSAGE``. */
full: boolean;
}
/** Manage the lifecycle of images attached to the Composer.
*
* Responsibilities in one place:
* - validation (MIME whitelist, count cap)
* - blob URL creation + revocation
* - Worker orchestration
* - focus bookkeeping so keyboard delete doesn't strand the user
*/
export function useAttachedImages(): UseAttachedImagesApi {
const [images, setImages] = useState<AttachedImage[]>([]);
// Ref mirror so ``enqueue`` can see the authoritative length when invoked
// multiple times in a single tick (rapid file selection, drag of many
// files, paste storms). ``state`` is stale for that second + call.
const imagesRef = useRef<AttachedImage[]>([]);
imagesRef.current = images;
const setEntry = useCallback((id: string, patch: Partial<AttachedImage>) => {
setImages((prev) => {
const next = prev.map((img) => (img.id === id ? { ...img, ...patch } : img));
imagesRef.current = next;
return next;
});
}, []);
const enqueue = useCallback(
(files: Iterable<File>) => {
const rejected: Array<{ file: File; reason: AttachmentError }> = [];
const toAdd: AttachedImage[] = [];
let slot = MAX_IMAGES_PER_MESSAGE - imagesRef.current.length;
for (const file of files) {
if (!ACCEPTED_MIMES.has(file.type)) {
rejected.push({ file, reason: "unsupported_type" });
continue;
}
if (slot <= 0) {
rejected.push({ file, reason: "too_many_images" });
continue;
}
slot -= 1;
toAdd.push({
id: uuid(),
file,
previewUrl: URL.createObjectURL(file),
status: "encoding",
});
}
if (toAdd.length > 0) {
const next = [...imagesRef.current, ...toAdd];
imagesRef.current = next;
setImages(next);
// Fire the Worker after the commit so chips render first (good INP).
for (const entry of toAdd) {
queueMicrotask(() => {
encodeImage(entry.file).then(
(result) => {
if (result.ok) {
setEntry(entry.id, {
status: "ready",
dataUrl: result.dataUrl,
encodedBytes: result.bytes,
normalized: result.normalized,
});
} else {
setEntry(entry.id, {
status: "error",
error: mapEncodeFailure(result.reason),
});
}
},
() => {
setEntry(entry.id, {
status: "error",
error: "decode_failed",
});
},
);
});
}
}
return { rejected };
},
[setEntry],
);
const remove = useCallback((id: string) => {
let nextFocusId: string | null = null;
setImages((prev) => {
const idx = prev.findIndex((img) => img.id === id);
if (idx === -1) return prev;
const target = prev[idx];
try {
URL.revokeObjectURL(target.previewUrl);
} catch {
// No-op: previewUrl revocation is best-effort.
}
const next = [...prev.slice(0, idx), ...prev.slice(idx + 1)];
imagesRef.current = next;
// Prefer moving focus to the chip at the same index, else previous.
const candidate = next[idx] ?? next[idx - 1];
nextFocusId = candidate?.id ?? null;
return next;
});
return { nextFocusId };
}, []);
const clear = useCallback(() => {
setImages((prev) => {
for (const img of prev) {
try {
URL.revokeObjectURL(img.previewUrl);
} catch {
// revoke is best-effort
}
}
imagesRef.current = [];
return [];
});
}, []);
// Final safety net: revoke any outstanding blob URLs on unmount. Safe
// under StrictMode double-invoke because revoked blob URLs are only
// referenced from in-hook chip state, which is rebuilt on remount.
useEffect(() => {
return () => {
for (const img of imagesRef.current) {
try {
URL.revokeObjectURL(img.previewUrl);
} catch {
// best-effort cleanup on unmount
}
}
};
}, []);
const encoding = images.some((img) => img.status === "encoding");
const full = images.length >= MAX_IMAGES_PER_MESSAGE;
return { images, enqueue, remove, clear, encoding, full };
}
+111
View File
@@ -0,0 +1,111 @@
import { useCallback, useRef, useState } from "react";
/** Extract image ``File``s from a paste / drop event.
*
* Deliberate behaviour:
* - Only items whose ``kind === "file"`` and ``type`` starts with
* ``image/`` are returned; ``<img>`` tags inside HTML fragments are
* ignored (defending against remote URL fetch + XSS surfaces).
* - Plain text pasted alongside images is *not* consumed by this helper,
* so the caller can still let the textarea receive it naturally.
*/
export function extractImageFilesFromPaste(
event: ClipboardEvent | React.ClipboardEvent,
): File[] {
const clipboard = (event as ClipboardEvent).clipboardData
?? (event as React.ClipboardEvent).clipboardData;
if (!clipboard) return [];
const files: File[] = [];
for (const item of Array.from(clipboard.items)) {
if (item.kind !== "file") continue;
if (!item.type.startsWith("image/")) continue;
const file = item.getAsFile();
if (file) files.push(file);
}
return files;
}
/** Extract dropped image files, mirroring ``extractImageFilesFromPaste``. */
export function extractImageFilesFromDrop(
event: DragEvent | React.DragEvent,
): File[] {
const dt = (event as DragEvent).dataTransfer
?? (event as React.DragEvent).dataTransfer;
if (!dt) return [];
const files: File[] = [];
for (const item of Array.from(dt.files)) {
if (item.type.startsWith("image/")) files.push(item);
}
return files;
}
export interface UseClipboardAndDropApi {
/** Whether a drag is currently hovering the drop zone (toggle dragover UI). */
isDragging: boolean;
onPaste: (
event: React.ClipboardEvent,
) => void;
onDragEnter: (event: React.DragEvent) => void;
onDragOver: (event: React.DragEvent) => void;
onDragLeave: (event: React.DragEvent) => void;
onDrop: (event: React.DragEvent) => void;
}
/** Wire paste + drag-and-drop to a callback.
*
* The hook owns ``isDragging`` state and the refcount that keeps it accurate
* across nested ``dragenter`` / ``dragleave`` events (a known DOM gotcha: the
* text cursor inside a textarea fires ``dragleave`` on entry, flicking the
* highlight off otherwise). */
export function useClipboardAndDrop(
onImageFiles: (files: File[]) => void,
): UseClipboardAndDropApi {
const [isDragging, setIsDragging] = useState(false);
const dragDepth = useRef(0);
const onPaste = useCallback(
(event: React.ClipboardEvent) => {
const files = extractImageFilesFromPaste(event);
if (files.length === 0) return;
// Consume only when an image is actually present; plain-text paste still
// reaches the textarea unmolested.
event.preventDefault();
onImageFiles(files);
},
[onImageFiles],
);
const onDragEnter = useCallback((event: React.DragEvent) => {
if (!Array.from(event.dataTransfer.types ?? []).includes("Files")) return;
event.preventDefault();
dragDepth.current += 1;
setIsDragging(true);
}, []);
const onDragOver = useCallback((event: React.DragEvent) => {
if (!Array.from(event.dataTransfer.types ?? []).includes("Files")) return;
event.preventDefault();
event.dataTransfer.dropEffect = "copy";
}, []);
const onDragLeave = useCallback((event: React.DragEvent) => {
if (!Array.from(event.dataTransfer.types ?? []).includes("Files")) return;
event.preventDefault();
dragDepth.current = Math.max(0, dragDepth.current - 1);
if (dragDepth.current === 0) setIsDragging(false);
}, []);
const onDrop = useCallback(
(event: React.DragEvent) => {
dragDepth.current = 0;
setIsDragging(false);
const files = extractImageFilesFromDrop(event);
if (files.length === 0) return;
event.preventDefault();
onImageFiles(files);
},
[onImageFiles],
);
return { isDragging, onPaste, onDragEnter, onDragOver, onDragLeave, onDrop };
}
+56 -7
View File
@@ -1,7 +1,13 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { useClient } from "@/providers/ClientProvider";
import type { InboundEvent, UIMessage } from "@/lib/types";
import type { StreamError } from "@/lib/nanobot-client";
import type {
InboundEvent,
OutboundMedia,
UIImage,
UIMessage,
} from "@/lib/types";
interface StreamBuffer {
/** ID of the assistant message currently receiving deltas. */
@@ -16,24 +22,52 @@ interface StreamBuffer {
* separately (e.g. via ``fetchSessionMessages``) since the server only replays
* live events.
*/
/** Payload passed to ``send`` when the user attaches one or more images.
*
* ``media`` is handed to the wire client verbatim; ``preview`` powers the
* optimistic user bubble (blob URLs so the preview appears before the server
* acks the frame). Keeping the two separate lets the bubble re-use the local
* blob URL even after the server persists the file under a different name. */
export interface SendImage {
media: OutboundMedia;
preview: UIImage;
}
export function useNanobotStream(
chatId: string | null,
initialMessages: UIMessage[] = [],
): {
messages: UIMessage[];
isStreaming: boolean;
send: (content: string) => void;
send: (content: string, images?: SendImage[]) => void;
setMessages: React.Dispatch<React.SetStateAction<UIMessage[]>>;
/** Latest transport-level fault raised since the last ``dismissStreamError``.
* ``null`` when there is nothing to show. */
streamError: StreamError | null;
/** Clear the current ``streamError`` (e.g. after the user dismisses the
* notification or starts a fresh action). */
dismissStreamError: () => void;
} {
const { client } = useClient();
const [messages, setMessages] = useState<UIMessage[]>(initialMessages);
const [isStreaming, setIsStreaming] = useState(false);
const [streamError, setStreamError] = useState<StreamError | null>(null);
const buffer = useRef<StreamBuffer | null>(null);
// Reset local state when switching chats.
useEffect(() => {
return client.onError((err) => setStreamError(err));
}, [client]);
const dismissStreamError = useCallback(() => setStreamError(null), []);
// Reset local state when switching chats. ``streamError`` is scoped to the
// send that triggered it, so a chat swap should wipe it out: a stale
// "Message too large" banner on a freshly-opened chat-B would confuse the
// user about which send actually failed (and in which chat).
useEffect(() => {
setMessages(initialMessages);
setIsStreaming(false);
setStreamError(null);
buffer.current = null;
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [chatId]);
@@ -145,8 +179,14 @@ export function useNanobotStream(
}, [chatId, client]);
const send = useCallback(
(content: string) => {
if (!chatId || !content.trim()) return;
(content: string, images?: SendImage[]) => {
if (!chatId) return;
const hasImages = !!images && images.length > 0;
// Text is optional when images are attached — the agent will still see
// the image blocks via ``media`` paths.
if (!hasImages && !content.trim()) return;
const previews = hasImages ? images!.map((i) => i.preview) : undefined;
setMessages((prev) => [
...prev,
{
@@ -154,12 +194,21 @@ export function useNanobotStream(
role: "user",
content,
createdAt: Date.now(),
...(previews ? { images: previews } : {}),
},
]);
client.sendMessage(chatId, content);
const wireMedia = hasImages ? images!.map((i) => i.media) : undefined;
client.sendMessage(chatId, content, wireMedia);
},
[chatId, client],
);
return { messages, isStreaming, send, setMessages };
return {
messages,
isStreaming,
send,
setMessages,
streamError,
dismissStreamError,
};
}
+13
View File
@@ -123,12 +123,25 @@ export function useSessionHistory(key: string | null): {
const ui: UIMessage[] = body.messages.flatMap((m, idx) => {
if (m.role !== "user" && m.role !== "assistant") return [];
if (typeof m.content !== "string") return [];
// Hydrate signed media URLs into the bubble's ``images`` slot so
// historical user turns render real previews (the live-send path
// uses data URLs; both shapes converge on the same ``UIImage``).
const images =
m.role === "user" &&
Array.isArray(m.media_urls) &&
m.media_urls.length > 0
? m.media_urls.map((mu) => ({
url: mu.url,
name: mu.name,
}))
: undefined;
return [
{
id: `hist-${idx}`,
role: m.role,
content: m.content,
createdAt: m.timestamp ? Date.parse(m.timestamp) : Date.now(),
...(images ? { images } : {}),
},
];
});
+31 -2
View File
@@ -64,7 +64,19 @@
"placeholderOpening": "Opening a new chat…",
"inputAria": "Message input",
"sendHint": "Enter to send · Shift+Enter for newline",
"send": "Send message"
"send": "Send message",
"attachImage": "Attach image",
"encoding": "Encoding…",
"remove": "Remove attachment",
"normalizedSizeHint": "{{orig}} → {{current}} (auto)",
"imageRejected": {
"unsupported_type": "Unsupported file type",
"too_many_images": "Max {{max}} images per message",
"magic_mismatch": "File doesn't look like a real image",
"decode_failed": "Couldn't decode this image",
"too_large": "Image is too large — try a smaller one",
"io": "Couldn't read this file"
}
},
"scrollToBottom": "Scroll to bottom"
},
@@ -72,12 +84,29 @@
"streaming": "streaming",
"assistantTyping": "Assistant is typing",
"toolSingle": "Using a tool",
"toolMany": "Used {{count}} tools"
"toolMany": "Used {{count}} tools",
"imageAttachment": "Image attachment"
},
"lightbox": {
"title": "Image preview",
"open": "View image",
"prev": "Previous image",
"next": "Next image",
"close": "Close image preview"
},
"code": {
"fallbackLanguage": "code",
"copyAria": "Copy code",
"copy": "Copy",
"copied": "Copied"
},
"common": {
"dismiss": "Dismiss"
},
"errors": {
"messageTooBig": {
"title": "Message too large",
"body": "The server rejected your last message because it exceeded the size limit. Remove some images or try smaller files, then send again."
}
}
}
+31 -2
View File
@@ -64,7 +64,19 @@
"placeholderOpening": "Abriendo un nuevo chat…",
"inputAria": "Entrada de mensaje",
"sendHint": "Enter para enviar · Shift+Enter para nueva línea",
"send": "Enviar mensaje"
"send": "Enviar mensaje",
"attachImage": "Adjuntar imagen",
"encoding": "Procesando…",
"remove": "Quitar adjunto",
"normalizedSizeHint": "{{orig}} → {{current}} (auto)",
"imageRejected": {
"unsupported_type": "Tipo de archivo no compatible",
"too_many_images": "Máximo {{max}} imágenes por mensaje",
"magic_mismatch": "El archivo no parece una imagen real",
"decode_failed": "No se pudo decodificar esta imagen",
"too_large": "Imagen demasiado grande — prueba una más pequeña",
"io": "No se pudo leer este archivo"
}
},
"scrollToBottom": "Desplazarse al final"
},
@@ -72,12 +84,29 @@
"streaming": "transmitiendo",
"assistantTyping": "El asistente está escribiendo",
"toolSingle": "Usando una herramienta",
"toolMany": "Se usaron {{count}} herramientas"
"toolMany": "Se usaron {{count}} herramientas",
"imageAttachment": "Imagen adjunta"
},
"lightbox": {
"title": "Vista previa de imagen",
"open": "Ver imagen",
"prev": "Imagen anterior",
"next": "Imagen siguiente",
"close": "Cerrar vista previa"
},
"code": {
"fallbackLanguage": "código",
"copyAria": "Copiar código",
"copy": "Copiar",
"copied": "Copiado"
},
"common": {
"dismiss": "Cerrar"
},
"errors": {
"messageTooBig": {
"title": "Mensaje demasiado grande",
"body": "El servidor rechazó tu último mensaje por superar el tamaño permitido. Quita algunas imágenes o usa archivos más pequeños y vuelve a enviarlo."
}
}
}
+31 -2
View File
@@ -64,7 +64,19 @@
"placeholderOpening": "Ouverture dune nouvelle discussion…",
"inputAria": "Champ de message",
"sendHint": "Entrée pour envoyer · Maj+Entrée pour un retour à la ligne",
"send": "Envoyer le message"
"send": "Envoyer le message",
"attachImage": "Joindre une image",
"encoding": "Traitement…",
"remove": "Retirer la pièce jointe",
"normalizedSizeHint": "{{orig}} → {{current}} (auto)",
"imageRejected": {
"unsupported_type": "Type de fichier non pris en charge",
"too_many_images": "Maximum {{max}} images par message",
"magic_mismatch": "Ce fichier n'est pas une image",
"decode_failed": "Impossible de décoder cette image",
"too_large": "Image trop grande — essayez-en une plus petite",
"io": "Impossible de lire ce fichier"
}
},
"scrollToBottom": "Faire défiler vers le bas"
},
@@ -72,12 +84,29 @@
"streaming": "en cours de génération",
"assistantTyping": "Lassistant est en train d’écrire",
"toolSingle": "Utilisation dun outil",
"toolMany": "{{count}} outils utilisés"
"toolMany": "{{count}} outils utilisés",
"imageAttachment": "Pièce jointe image"
},
"lightbox": {
"title": "Aperçu de limage",
"open": "Voir limage",
"prev": "Image précédente",
"next": "Image suivante",
"close": "Fermer laperçu"
},
"code": {
"fallbackLanguage": "code",
"copyAria": "Copier le code",
"copy": "Copier",
"copied": "Copié"
},
"common": {
"dismiss": "Fermer"
},
"errors": {
"messageTooBig": {
"title": "Message trop volumineux",
"body": "Le serveur a rejeté votre dernier message car il dépasse la taille autorisée. Retirez des images ou choisissez des fichiers plus légers, puis renvoyez-le."
}
}
}
+31 -2
View File
@@ -64,7 +64,19 @@
"placeholderOpening": "Membuka obrolan baru…",
"inputAria": "Input pesan",
"sendHint": "Enter untuk kirim · Shift+Enter untuk baris baru",
"send": "Kirim pesan"
"send": "Kirim pesan",
"attachImage": "Lampirkan gambar",
"encoding": "Memproses…",
"remove": "Hapus lampiran",
"normalizedSizeHint": "{{orig}} → {{current}} (auto)",
"imageRejected": {
"unsupported_type": "Tipe file tidak didukung",
"too_many_images": "Maksimal {{max}} gambar per pesan",
"magic_mismatch": "File ini tampaknya bukan gambar asli",
"decode_failed": "Tidak dapat mendekode gambar ini",
"too_large": "Gambar terlalu besar — coba yang lebih kecil",
"io": "Tidak dapat membaca file ini"
}
},
"scrollToBottom": "Gulir ke bawah"
},
@@ -72,12 +84,29 @@
"streaming": "sedang mengalir",
"assistantTyping": "Asisten sedang mengetik",
"toolSingle": "Menggunakan sebuah alat",
"toolMany": "Menggunakan {{count}} alat"
"toolMany": "Menggunakan {{count}} alat",
"imageAttachment": "Lampiran gambar"
},
"lightbox": {
"title": "Pratinjau gambar",
"open": "Lihat gambar",
"prev": "Gambar sebelumnya",
"next": "Gambar berikutnya",
"close": "Tutup pratinjau"
},
"code": {
"fallbackLanguage": "kode",
"copyAria": "Salin kode",
"copy": "Salin",
"copied": "Tersalin"
},
"common": {
"dismiss": "Tutup"
},
"errors": {
"messageTooBig": {
"title": "Pesan terlalu besar",
"body": "Server menolak pesan terakhir karena melebihi batas ukuran. Hapus beberapa gambar atau gunakan berkas yang lebih kecil, lalu coba lagi."
}
}
}
+31 -2
View File
@@ -64,7 +64,19 @@
"placeholderOpening": "新しいチャットを開いています…",
"inputAria": "メッセージ入力欄",
"sendHint": "Enter で送信 · Shift+Enter で改行",
"send": "メッセージを送信"
"send": "メッセージを送信",
"attachImage": "画像を添付",
"encoding": "処理中…",
"remove": "添付を削除",
"normalizedSizeHint": "{{orig}} → {{current}}(自動圧縮)",
"imageRejected": {
"unsupported_type": "対応していないファイル形式です",
"too_many_images": "1 メッセージにつき最大 {{max}} 枚です",
"magic_mismatch": "画像ファイルではないようです",
"decode_failed": "この画像をデコードできません",
"too_large": "画像が大きすぎます。小さいものを選んでください",
"io": "このファイルを読み込めません"
}
},
"scrollToBottom": "一番下へスクロール"
},
@@ -72,12 +84,29 @@
"streaming": "生成中",
"assistantTyping": "アシスタントが入力中",
"toolSingle": "ツールを使用中",
"toolMany": "{{count}} 個のツールを使用"
"toolMany": "{{count}} 個のツールを使用",
"imageAttachment": "画像の添付"
},
"lightbox": {
"title": "画像プレビュー",
"open": "画像を表示",
"prev": "前の画像",
"next": "次の画像",
"close": "プレビューを閉じる"
},
"code": {
"fallbackLanguage": "コード",
"copyAria": "コードをコピー",
"copy": "コピー",
"copied": "コピーしました"
},
"common": {
"dismiss": "閉じる"
},
"errors": {
"messageTooBig": {
"title": "メッセージが大きすぎます",
"body": "サイズ上限を超えたため、直前のメッセージはサーバーに拒否されました。画像を減らすか、より小さいファイルに差し替えて再送してください。"
}
}
}
+31 -2
View File
@@ -64,7 +64,19 @@
"placeholderOpening": "새 채팅을 여는 중…",
"inputAria": "메시지 입력",
"sendHint": "Enter로 전송 · Shift+Enter로 줄바꿈",
"send": "메시지 보내기"
"send": "메시지 보내기",
"attachImage": "이미지 첨부",
"encoding": "처리 중…",
"remove": "첨부 제거",
"normalizedSizeHint": "{{orig}} → {{current}} (자동 압축)",
"imageRejected": {
"unsupported_type": "지원하지 않는 파일 형식입니다",
"too_many_images": "메시지당 최대 {{max}}장까지 가능합니다",
"magic_mismatch": "이미지 파일이 아닌 것 같습니다",
"decode_failed": "이 이미지를 디코딩할 수 없습니다",
"too_large": "이미지가 너무 큽니다. 더 작은 걸로 시도해 주세요",
"io": "이 파일을 읽을 수 없습니다"
}
},
"scrollToBottom": "맨 아래로 스크롤"
},
@@ -72,12 +84,29 @@
"streaming": "생성 중",
"assistantTyping": "도우미가 입력 중",
"toolSingle": "도구 사용 중",
"toolMany": "도구 {{count}}개 사용됨"
"toolMany": "도구 {{count}}개 사용됨",
"imageAttachment": "이미지 첨부"
},
"lightbox": {
"title": "이미지 미리보기",
"open": "이미지 보기",
"prev": "이전 이미지",
"next": "다음 이미지",
"close": "미리보기 닫기"
},
"code": {
"fallbackLanguage": "코드",
"copyAria": "코드 복사",
"copy": "복사",
"copied": "복사됨"
},
"common": {
"dismiss": "닫기"
},
"errors": {
"messageTooBig": {
"title": "메시지가 너무 큽니다",
"body": "마지막 메시지가 서버의 크기 제한을 초과하여 거부되었습니다. 이미지를 줄이거나 더 작은 파일로 바꿔서 다시 보내 주세요."
}
}
}
+31 -2
View File
@@ -64,7 +64,19 @@
"placeholderOpening": "Đang mở cuộc trò chuyện mới…",
"inputAria": "Ô nhập tin nhắn",
"sendHint": "Enter để gửi · Shift+Enter để xuống dòng",
"send": "Gửi tin nhắn"
"send": "Gửi tin nhắn",
"attachImage": "Đính kèm ảnh",
"encoding": "Đang xử lý…",
"remove": "Xóa tệp đính kèm",
"normalizedSizeHint": "{{orig}} → {{current}} (tự động)",
"imageRejected": {
"unsupported_type": "Loại tệp không được hỗ trợ",
"too_many_images": "Tối đa {{max}} ảnh mỗi tin nhắn",
"magic_mismatch": "Tệp này không phải là một ảnh thực",
"decode_failed": "Không thể giải mã ảnh này",
"too_large": "Ảnh quá lớn — hãy thử ảnh nhỏ hơn",
"io": "Không thể đọc tệp này"
}
},
"scrollToBottom": "Cuộn xuống cuối"
},
@@ -72,12 +84,29 @@
"streaming": "đang truyền",
"assistantTyping": "Trợ lý đang nhập",
"toolSingle": "Đang dùng một công cụ",
"toolMany": "Đã dùng {{count}} công cụ"
"toolMany": "Đã dùng {{count}} công cụ",
"imageAttachment": "Tệp hình ảnh đính kèm"
},
"lightbox": {
"title": "Xem trước ảnh",
"open": "Xem ảnh",
"prev": "Ảnh trước",
"next": "Ảnh tiếp theo",
"close": "Đóng xem trước"
},
"code": {
"fallbackLanguage": "mã",
"copyAria": "Sao chép mã",
"copy": "Sao chép",
"copied": "Đã sao chép"
},
"common": {
"dismiss": "Đóng"
},
"errors": {
"messageTooBig": {
"title": "Tin nhắn quá lớn",
"body": "Máy chủ đã từ chối tin nhắn trước vì vượt quá giới hạn kích thước. Hãy bớt ảnh hoặc chọn tệp nhỏ hơn rồi thử lại."
}
}
}
+31 -2
View File
@@ -64,7 +64,19 @@
"placeholderOpening": "正在打开新对话…",
"inputAria": "消息输入框",
"sendHint": "Enter 发送 · Shift+Enter 换行",
"send": "发送消息"
"send": "发送消息",
"attachImage": "添加图片",
"encoding": "处理中…",
"remove": "移除附件",
"normalizedSizeHint": "{{orig}} → {{current}}(已自动压缩)",
"imageRejected": {
"unsupported_type": "不支持的文件类型",
"too_many_images": "每条消息最多 {{max}} 张图片",
"magic_mismatch": "文件看起来不像真实的图片",
"decode_failed": "无法解码这张图片",
"too_large": "图片太大,请换一张小一点的",
"io": "无法读取该文件"
}
},
"scrollToBottom": "滚动到底部"
},
@@ -72,12 +84,29 @@
"streaming": "流式输出中",
"assistantTyping": "助手正在输入",
"toolSingle": "正在使用工具",
"toolMany": "已使用 {{count}} 个工具"
"toolMany": "已使用 {{count}} 个工具",
"imageAttachment": "图片附件"
},
"lightbox": {
"title": "图片预览",
"open": "查看图片",
"prev": "上一张",
"next": "下一张",
"close": "关闭预览"
},
"code": {
"fallbackLanguage": "代码",
"copyAria": "复制代码",
"copy": "复制",
"copied": "已复制"
},
"common": {
"dismiss": "关闭"
},
"errors": {
"messageTooBig": {
"title": "消息过大",
"body": "服务端因超过大小限制拒收了上一条消息。可移除部分图片或使用更小的图片后重试。"
}
}
}
+31 -2
View File
@@ -64,7 +64,19 @@
"placeholderOpening": "正在開啟新對話…",
"inputAria": "訊息輸入框",
"sendHint": "Enter 送出 · Shift+Enter 換行",
"send": "送出訊息"
"send": "送出訊息",
"attachImage": "附加圖片",
"encoding": "處理中…",
"remove": "移除附件",
"normalizedSizeHint": "{{orig}} → {{current}}(已自動壓縮)",
"imageRejected": {
"unsupported_type": "不支援的檔案類型",
"too_many_images": "每則訊息最多 {{max}} 張圖片",
"magic_mismatch": "檔案看起來不像真正的圖片",
"decode_failed": "無法解碼這張圖片",
"too_large": "圖片太大,請換一張小一點的",
"io": "無法讀取這個檔案"
}
},
"scrollToBottom": "捲動到底部"
},
@@ -72,12 +84,29 @@
"streaming": "串流輸出中",
"assistantTyping": "助理正在輸入",
"toolSingle": "正在使用工具",
"toolMany": "已使用 {{count}} 個工具"
"toolMany": "已使用 {{count}} 個工具",
"imageAttachment": "圖片附件"
},
"lightbox": {
"title": "圖片預覽",
"open": "檢視圖片",
"prev": "上一張",
"next": "下一張",
"close": "關閉預覽"
},
"code": {
"fallbackLanguage": "程式碼",
"copyAria": "複製程式碼",
"copy": "複製",
"copied": "已複製"
},
"common": {
"dismiss": "關閉"
},
"errors": {
"messageTooBig": {
"title": "訊息過大",
"body": "伺服器因超過大小限制拒收了上一則訊息。可移除部分圖片或改用較小的圖片後再試。"
}
}
}
+13
View File
@@ -57,6 +57,16 @@ export async function listSessions(
}));
}
/** Signed image URL attached to a historical user message. The server
* emits these in place of raw on-disk paths so the client can render
* previews without learning where media lives on disk. Each URL is a
* self-authenticating ``/api/media/...`` route (see backend
* ``_sign_media_path``) safe to drop into an ``<img src>`` attribute. */
export interface SessionMediaUrl {
url: string;
name?: string;
}
export async function fetchSessionMessages(
token: string,
key: string,
@@ -72,6 +82,9 @@ export async function fetchSessionMessages(
tool_calls?: unknown;
tool_call_id?: string;
name?: string;
/** Present on ``user`` turns that attached images. Paths have already
* been stripped server-side; only the signed fetch URLs survive. */
media_urls?: SessionMediaUrl[];
}>;
}> {
return request(
+97
View File
@@ -0,0 +1,97 @@
/**
* Main-thread client for the image encoder Worker.
*
* Lazily boots a single ``imageEncode.worker`` and multiplexes requests onto
* it by a random request id. Falls back to an inline call when the Worker
* can't be constructed (tests, ancient browsers) so the Composer always has a
* working path.
*/
import {
encodeImageInWorker,
type EncodeResponse,
} from "@/workers/imageEncode.worker";
export type { EncodeResponse, EncodeSuccess, EncodeFailure } from "@/workers/imageEncode.worker";
export { TARGET_MAX_BYTES } from "@/workers/imageEncode.worker";
type Pending = {
resolve: (r: EncodeResponse) => void;
reject: (err: Error) => void;
};
let worker: Worker | null = null;
let bootAttempted = false;
const pending = new Map<string, Pending>();
function bootWorker(): Worker | null {
if (bootAttempted) return worker;
bootAttempted = true;
if (typeof Worker === "undefined") return null;
try {
worker = new Worker(
new URL("@/workers/imageEncode.worker.ts", import.meta.url),
{ type: "module" },
);
worker.addEventListener("message", (ev: MessageEvent<EncodeResponse>) => {
const entry = pending.get(ev.data.id);
if (!entry) return;
pending.delete(ev.data.id);
entry.resolve(ev.data);
});
worker.addEventListener("error", (ev) => {
// Cancel every in-flight request on a Worker crash.
for (const [, entry] of pending) {
entry.reject(new Error(`image encoder worker error: ${ev.message}`));
}
pending.clear();
worker?.terminate();
worker = null;
});
return worker;
} catch {
worker = null;
return null;
}
}
function newId(): string {
// ``crypto.randomUUID`` is widely available; fall back to Math.random if not.
if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
return (crypto as Crypto).randomUUID();
}
return `img-${Date.now()}-${Math.random().toString(36).slice(2)}`;
}
/** Encode ``file`` off the main thread when possible. Always resolves — errors
* are returned as ``{ok: false, reason}`` — so callers can render inline
* validation without wrapping in try/catch. */
export async function encodeImage(file: File): Promise<EncodeResponse> {
const id = newId();
const w = bootWorker();
if (!w) {
// Inline fallback: same logic, just on the main thread.
return encodeImageInWorker({ id, file });
}
return new Promise<EncodeResponse>((resolve, reject) => {
pending.set(id, { resolve, reject });
try {
w.postMessage({ id, file });
} catch (err) {
pending.delete(id);
reject(err instanceof Error ? err : new Error(String(err)));
}
});
}
/** Release the singleton Worker (tests / teardown). */
export function disposeImageEncoder(): void {
if (worker) {
worker.terminate();
worker = null;
}
bootAttempted = false;
for (const [, entry] of pending) {
entry.reject(new Error("image encoder disposed"));
}
pending.clear();
}
+60 -5
View File
@@ -1,4 +1,9 @@
import type { ConnectionStatus, InboundEvent, Outbound } from "./types";
import type {
ConnectionStatus,
InboundEvent,
Outbound,
OutboundMedia,
} from "./types";
/** WebSocket readyState constants, referenced by value to stay portable
* across runtimes that don't expose a global ``WebSocket`` (tests, SSR). */
@@ -9,6 +14,22 @@ type Unsubscribe = () => void;
type EventHandler = (ev: InboundEvent) => void;
type StatusHandler = (status: ConnectionStatus) => void;
/** Structured connection-level errors surfaced to the UI.
*
* These are *not* InboundEvent errors from the server application layer —
* those arrive as ``{event: "error"}`` messages via ``onChat``. These are
* transport-level or protocol-level faults the UI should make visible so
* the user understands *why* their action failed (as opposed to silently
* reconnecting under the hood).
*/
export type StreamError =
/** Server rejected the inbound frame as too large (WS close code 1009).
* Typically means the user attached images whose base64 size exceeded
* ``maxMessageBytes`` on the server. */
| { kind: "message_too_big" };
type ErrorHandler = (error: StreamError) => void;
interface PendingNewChat {
resolve: (chatId: string) => void;
reject: (err: Error) => void;
@@ -36,6 +57,7 @@ export interface NanobotClientOptions {
export class NanobotClient {
private socket: WebSocket | null = null;
private statusHandlers = new Set<StatusHandler>();
private errorHandlers = new Set<ErrorHandler>();
// chat_id -> handlers listening on it
private chatHandlers = new Map<string, Set<EventHandler>>();
// chat_ids we've attached to since connect; re-attached after reconnects
@@ -84,6 +106,14 @@ export class NanobotClient {
};
}
/** Subscribe to transport-level faults (see :type:`StreamError`). */
onError(handler: ErrorHandler): Unsubscribe {
this.errorHandlers.add(handler);
return () => {
this.errorHandlers.delete(handler);
};
}
/** Subscribe to events for a given chat_id. Auto-attaches on the next open. */
onChat(chatId: string, handler: EventHandler): Unsubscribe {
let handlers = this.chatHandlers.get(chatId);
@@ -110,7 +140,7 @@ export class NanobotClient {
sock.onopen = () => this.handleOpen();
sock.onmessage = (ev) => this.handleMessage(ev);
sock.onerror = () => this.setStatus("error");
sock.onclose = () => this.handleClose();
sock.onclose = (ev) => this.handleClose(ev);
}
close(): void {
@@ -151,9 +181,13 @@ export class NanobotClient {
}
}
sendMessage(chatId: string, content: string): void {
sendMessage(chatId: string, content: string, media?: OutboundMedia[]): void {
this.knownChats.add(chatId);
this.queueSend({ type: "message", chat_id: chatId, content });
const frame: Outbound =
media && media.length > 0
? { type: "message", chat_id: chatId, content, media }
: { type: "message", chat_id: chatId, content };
this.queueSend(frame);
}
// -- internals ---------------------------------------------------------
@@ -211,13 +245,20 @@ export class NanobotClient {
for (const h of handlers) h(ev);
}
private handleClose(): void {
private handleClose(event?: { code?: number }): void {
this.socket = null;
if (this.pendingNewChat) {
clearTimeout(this.pendingNewChat.timer);
this.pendingNewChat.reject(new Error("socket closed"));
this.pendingNewChat = null;
}
// Surface structured reasons *before* reconnect logic so the UI can
// display the error even while the client transparently reconnects.
// Browsers populate ``CloseEvent.code`` with the wire-level close code;
// 1009 = Message Too Big (server's max frame guard).
if (event?.code === 1009) {
this.emitError({ kind: "message_too_big" });
}
if (this.intentionallyClosed || !this.shouldReconnect) {
this.setStatus("closed");
return;
@@ -225,6 +266,20 @@ export class NanobotClient {
this.scheduleReconnect();
}
private emitError(error: StreamError): void {
// Isolate subscribers so a throwing handler cannot abort the surrounding
// ``handleClose`` flow (which still owes us a reconnect decision + status
// update). We deliberately swallow here: error reporting is best-effort
// and must never be allowed to compound the failure it's reporting.
for (const handler of this.errorHandlers) {
try {
handler(error);
} catch {
// best-effort: subscriber fault must not stall transport bookkeeping
}
}
}
private scheduleReconnect(): void {
this.setStatus("reconnecting");
const attempt = this.reconnectAttempts++;
+39 -1
View File
@@ -4,6 +4,24 @@ export type Role = "user" | "assistant" | "tool" | "system";
* progress pings) that should not be rendered as conversational replies. */
export type MessageKind = "message" | "trace";
/** One image attached to a UIMessage.
*
* ``url`` can arrive in three different shapes, which the bubble renders
* identically:
* - A ``data:image/...;base64,...`` URL generated by the Composer for the
* optimistic preview of an in-flight user turn. Self-contained, no
* lifecycle.
* - A signed ``/api/media/...`` URL attached to a historical user turn by
* the backend on session replay. Safe to drop into an ``<img src>``.
* - Absent. The backend couldn't resolve a stored path (file moved,
* deleted, or pre-media-persistence session). The bubble shows a
* placeholder tile with ``name`` as the label.
*/
export interface UIImage {
url?: string;
name?: string;
}
export interface UIMessage {
id: string;
role: Role;
@@ -14,6 +32,8 @@ export interface UIMessage {
/** For trace rows: each individual hint line, so consecutive hints can
* render as a single collapsible group. */
traces?: string[];
/** User turn: optimistic blob URLs for preview. Replay: placeholder chips. */
images?: UIImage[];
}
export interface ChatSummary {
@@ -68,7 +88,25 @@ export type InboundEvent =
}
| { event: "error"; chat_id?: string; detail?: string };
/** Base64-encoded image attached to an outbound ``message`` envelope.
*
* ``data_url`` must be a ``data:image/<png|jpeg|webp|gif>;base64,...`` string
* — the server whitelists those MIME types and rejects everything else
* (including SVG, to avoid an XSS surface). ``name`` is advisory: it's
* preserved for the file on disk and surfaced as the placeholder label when
* the session is replayed.
*/
export interface OutboundMedia {
data_url: string;
name?: string;
}
export type Outbound =
| { type: "new_chat" }
| { type: "attach"; chat_id: string }
| { type: "message"; chat_id: string; content: string };
| {
type: "message";
chat_id: string;
content: string;
media?: OutboundMedia[];
};
+1
View File
@@ -53,6 +53,7 @@ vi.mock("@/lib/nanobot-client", () => {
defaultChatId: string | null = null;
connect = connectSpy;
onStatus = () => () => {};
onError = () => () => {};
onChat = () => () => {};
sendMessage = vi.fn();
newChat = vi.fn();
+97 -1
View File
@@ -20,7 +20,7 @@ class FakeSocket {
onopen: (() => void) | null = null;
onmessage: ((ev: MessageEvent) => void) | null = null;
onerror: (() => void) | null = null;
onclose: (() => void) | null = null;
onclose: ((ev?: { code?: number }) => void) | null = null;
constructor(url: string) {
this.url = url;
@@ -36,6 +36,13 @@ class FakeSocket {
this.onclose?.();
}
/** Simulate a server-initiated drop with a specific wire-level close code
* (e.g. ``1009`` for Message Too Big). */
fakeCloseWithCode(code: number) {
this.readyState = FakeSocket.CLOSED;
this.onclose?.({ code });
}
fakeOpen() {
this.readyState = FakeSocket.OPEN;
this.onopen?.();
@@ -172,6 +179,95 @@ describe("NanobotClient", () => {
expect(seen.at(-1)).toBe("closed");
});
it("passes media through into the message envelope", () => {
const client = new NanobotClient({
url: "ws://test",
reconnect: false,
socketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
});
client.connect();
lastSocket().fakeOpen();
client.sendMessage("chat-x", "look", [
{ data_url: "data:image/png;base64,AAAA", name: "shot.png" },
]);
const lastFrame = JSON.parse(lastSocket().sent.at(-1) as string);
expect(lastFrame).toEqual({
type: "message",
chat_id: "chat-x",
content: "look",
media: [{ data_url: "data:image/png;base64,AAAA", name: "shot.png" }],
});
});
it("omits media from the envelope when no images are attached", () => {
const client = new NanobotClient({
url: "ws://test",
reconnect: false,
socketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
});
client.connect();
lastSocket().fakeOpen();
client.sendMessage("chat-x", "hello");
const lastFrame = JSON.parse(lastSocket().sent.at(-1) as string);
expect(lastFrame).not.toHaveProperty("media");
expect(lastFrame).toEqual({
type: "message",
chat_id: "chat-x",
content: "hello",
});
});
it("emits a message_too_big error when the socket closes with code 1009", () => {
const client = new NanobotClient({
url: "ws://test",
reconnect: false,
socketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
});
const errors: Array<{ kind: string }> = [];
client.onError((e) => errors.push(e));
client.connect();
lastSocket().fakeOpen();
// Server rejected an outbound frame as too large.
lastSocket().fakeCloseWithCode(1009);
expect(errors).toEqual([{ kind: "message_too_big" }]);
});
it("isolates throwing error handlers so reconnect bookkeeping still runs", async () => {
const client = new NanobotClient({
url: "ws://test",
reconnect: true,
maxBackoffMs: 5,
socketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
});
// First handler explodes; subsequent reconnect state must be untouched.
client.onError(() => {
throw new Error("subscriber blew up");
});
const seenStatuses: string[] = [];
client.onStatus((s) => seenStatuses.push(s));
client.connect();
lastSocket().fakeOpen();
lastSocket().fakeCloseWithCode(1009);
// Despite the throwing handler, the client must still schedule a reconnect.
expect(seenStatuses).toContain("reconnecting");
await vi.advanceTimersByTimeAsync(20);
expect(FakeSocket.instances.length).toBeGreaterThan(1);
});
it("does not emit a stream error on a vanilla socket close", () => {
const client = new NanobotClient({
url: "ws://test",
reconnect: false,
socketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
});
const errors: Array<{ kind: string }> = [];
client.onError((e) => errors.push(e));
client.connect();
lastSocket().fakeOpen();
lastSocket().close();
expect(errors).toEqual([]);
});
it("surfaces 'reconnecting' only on an unexpected drop", async () => {
const client = new NanobotClient({
url: "ws://test",
@@ -0,0 +1,172 @@
import {
act,
fireEvent,
render,
screen,
waitFor,
} from "@testing-library/react";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { ThreadComposer } from "@/components/thread/ThreadComposer";
import type { EncodeResponse } from "@/lib/imageEncode";
const encodeImage = vi.fn<[File], Promise<EncodeResponse>>();
vi.mock("@/lib/imageEncode", async (importOriginal) => {
const actual = await importOriginal<typeof import("@/lib/imageEncode")>();
return {
...actual,
encodeImage: (file: File) => encodeImage(file),
};
});
function pngFile(name = "a.png", size = 10) {
return new File([new Uint8Array(size)], name, { type: "image/png" });
}
function resolveReady(file: File): EncodeResponse {
return {
id: "stub",
ok: true,
dataUrl: `data:image/png;base64,${btoa(file.name)}`,
mimeType: "image/png",
bytes: file.size,
normalized: false,
} as EncodeResponse;
}
beforeEach(() => {
encodeImage.mockReset();
let id = 0;
// Tests never read the preview URL contents so a stable blob: stub is fine.
if (!(globalThis.URL as unknown as { createObjectURL?: unknown }).createObjectURL) {
(globalThis.URL as unknown as { createObjectURL: (b: Blob) => string }).createObjectURL =
() => `blob:mock/${++id}`;
}
if (!(globalThis.URL as unknown as { revokeObjectURL?: unknown }).revokeObjectURL) {
(globalThis.URL as unknown as { revokeObjectURL: (u: string) => void }).revokeObjectURL =
() => {};
}
});
describe("ThreadComposer — image attachments", () => {
it("attaches a picked image and includes its data url on send", async () => {
const file = pngFile("a.png");
encodeImage.mockResolvedValueOnce(resolveReady(file));
const onSend = vi.fn();
render(<ThreadComposer onSend={onSend} />);
const input = screen
.getByLabelText(/message input/i)
.closest("form")!
.querySelector('input[type="file"]') as HTMLInputElement;
await act(async () => {
fireEvent.change(input, { target: { files: [file] } });
});
await waitFor(() =>
expect(screen.getByTestId("composer-chip")).toBeInTheDocument(),
);
const textarea = screen.getByLabelText(/message input/i);
fireEvent.change(textarea, { target: { value: "hi" } });
fireEvent.keyDown(textarea, { key: "Enter" });
expect(onSend).toHaveBeenCalledTimes(1);
const [content, images] = onSend.mock.calls[0];
expect(content).toBe("hi");
expect(images).toHaveLength(1);
expect(images[0].media.data_url).toContain("data:image/png;base64,");
expect(images[0].media.name).toBe("a.png");
});
it("blocks send while an image is still encoding", async () => {
const file = pngFile("slow.png");
let resolveEncode: (r: EncodeResponse) => void = () => {};
encodeImage.mockReturnValueOnce(
new Promise((r) => {
resolveEncode = r;
}),
);
const onSend = vi.fn();
render(<ThreadComposer onSend={onSend} />);
const fileInput = screen
.getByLabelText(/message input/i)
.closest("form")!
.querySelector('input[type="file"]') as HTMLInputElement;
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
const textarea = screen.getByLabelText(/message input/i);
fireEvent.change(textarea, { target: { value: "hello" } });
fireEvent.keyDown(textarea, { key: "Enter" });
expect(onSend).not.toHaveBeenCalled();
await act(async () => {
resolveEncode(resolveReady(file));
await Promise.resolve();
});
fireEvent.keyDown(textarea, { key: "Enter" });
expect(onSend).toHaveBeenCalledTimes(1);
});
it("rejects a non-image paste silently without adding a chip", async () => {
const onSend = vi.fn();
render(<ThreadComposer onSend={onSend} />);
const textarea = screen.getByLabelText(/message input/i);
fireEvent.paste(textarea, {
clipboardData: {
files: [],
items: [
{
kind: "string",
type: "text/plain",
getAsFile: () => null,
},
],
types: ["text/plain"],
getData: () => "some pasted text",
},
});
expect(screen.queryByTestId("composer-chip")).toBeNull();
expect(encodeImage).not.toHaveBeenCalled();
});
it("surfaces an inline error when encoding fails", async () => {
const file = pngFile("bad.png");
encodeImage.mockResolvedValueOnce({
id: "stub",
ok: false,
reason: "decode_failed",
} as EncodeResponse);
const onSend = vi.fn();
render(<ThreadComposer onSend={onSend} />);
const fileInput = screen
.getByLabelText(/message input/i)
.closest("form")!
.querySelector('input[type="file"]') as HTMLInputElement;
await act(async () => {
fireEvent.change(fileInput, { target: { files: [file] } });
});
await waitFor(() => {
const chip = screen.getByTestId("composer-chip");
expect(chip.textContent ?? "").toMatch(/decode|image/i);
});
const textarea = screen.getByLabelText(/message input/i);
fireEvent.change(textarea, { target: { value: "hi" } });
fireEvent.keyDown(textarea, { key: "Enter" });
expect(onSend).not.toHaveBeenCalled();
});
});
+90
View File
@@ -6,11 +6,21 @@ import { ThreadShell } from "@/components/thread/ThreadShell";
import { ClientProvider } from "@/providers/ClientProvider";
function makeClient() {
const errorHandlers = new Set<(err: { kind: string }) => void>();
return {
status: "open" as const,
defaultChatId: null as string | null,
onStatus: () => () => {},
onChat: () => () => {},
onError: (handler: (err: { kind: string }) => void) => {
errorHandlers.add(handler);
return () => {
errorHandlers.delete(handler);
};
},
_emitError(err: { kind: string }) {
for (const h of errorHandlers) h(err);
},
sendMessage: vi.fn(),
newChat: vi.fn(),
attach: vi.fn(),
@@ -88,6 +98,7 @@ describe("ThreadShell", () => {
expect(client.sendMessage).toHaveBeenCalledWith(
"chat-a",
"persist me across tabs",
undefined,
),
);
expect(screen.getByText("persist me across tabs")).toBeInTheDocument();
@@ -151,6 +162,7 @@ describe("ThreadShell", () => {
expect(client.sendMessage).toHaveBeenCalledWith(
"chat-a",
"delete me cleanly",
undefined,
),
);
expect(screen.getByText("delete me cleanly")).toBeInTheDocument();
@@ -241,6 +253,84 @@ describe("ThreadShell", () => {
expect(screen.queryByText("old answer")).not.toBeInTheDocument();
});
it("surfaces a dismissible banner when the stream reports message_too_big", async () => {
const client = makeClient();
const onNewChat = vi.fn().mockResolvedValue("chat-a");
render(
wrap(
client,
<ThreadShell
session={session("chat-a")}
title="Chat chat-a"
onToggleSidebar={() => {}}
onGoHome={() => {}}
onNewChat={onNewChat}
/>,
),
);
// No banner yet: only appears once the client emits a matching error.
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
await act(async () => {
client._emitError({ kind: "message_too_big" });
});
const banner = await screen.findByRole("alert");
expect(banner).toHaveTextContent("Message too large");
fireEvent.click(screen.getByRole("button", { name: "Dismiss" }));
await waitFor(() => {
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
});
});
it("clears the stream error banner when the user switches to another chat", async () => {
const client = makeClient();
const onNewChat = vi.fn().mockResolvedValue("chat-a");
const { rerender } = render(
wrap(
client,
<ThreadShell
session={session("chat-a")}
title="Chat chat-a"
onToggleSidebar={() => {}}
onGoHome={() => {}}
onNewChat={onNewChat}
/>,
),
);
await act(async () => {
client._emitError({ kind: "message_too_big" });
});
expect(await screen.findByRole("alert")).toBeInTheDocument();
// Switch to a different chat. The banner was about the *previous* send
// in chat-a; it must not leak into chat-b's view.
await act(async () => {
rerender(
wrap(
client,
<ThreadShell
session={session("chat-b")}
title="Chat chat-b"
onToggleSidebar={() => {}}
onGoHome={() => {}}
onNewChat={onNewChat}
/>,
),
);
});
await waitFor(() => {
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
});
});
it("clears the previous thread immediately while the next session loads", async () => {
const client = makeClient();
const onNewChat = vi.fn().mockResolvedValue("chat-b");
@@ -13,6 +13,7 @@ function fakeClient() {
status: "open" as const,
defaultChatId: null as string | null,
onStatus: () => () => {},
onError: () => () => {},
onChat(chatId: string, h: (ev: InboundEvent) => void) {
let set = handlers.get(chatId);
if (!set) {
+51 -1
View File
@@ -2,7 +2,7 @@ import { act, renderHook, waitFor } from "@testing-library/react";
import type { ReactNode } from "react";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { useSessions } from "@/hooks/useSessions";
import { useSessionHistory, useSessions } from "@/hooks/useSessions";
import * as api from "@/lib/api";
import { ClientProvider } from "@/providers/ClientProvider";
@@ -21,6 +21,7 @@ function fakeClient() {
status: "open" as const,
defaultChatId: null as string | null,
onStatus: () => () => {},
onError: () => () => {},
onChat: () => () => {},
sendMessage: vi.fn(),
newChat: vi.fn(),
@@ -86,6 +87,55 @@ describe("useSessions", () => {
expect(result.current.sessions.map((s) => s.key)).toEqual(["websocket:chat-b"]);
});
it("hydrates media_urls from historical user turns into UIMessage.images", async () => {
// Round-trip check for the signed-media replay: the backend emits
// ``media_urls`` on a historical user row and the hook must surface them
// as ``images`` so the bubble can render the preview. Assistant turns
// carry no media_urls and should not sprout an ``images`` field.
vi.mocked(api.fetchSessionMessages).mockResolvedValue({
key: "websocket:chat-media",
created_at: "2026-04-20T10:00:00Z",
updated_at: "2026-04-20T10:05:00Z",
messages: [
{
role: "user",
content: "what's this?",
timestamp: "2026-04-20T10:00:00Z",
media_urls: [
{ url: "/api/media/sig-1/payload-1", name: "snap.png" },
{ url: "/api/media/sig-2/payload-2", name: "diag.jpg" },
],
},
{
role: "assistant",
content: "it's a cat",
timestamp: "2026-04-20T10:00:01Z",
},
{
role: "user",
content: "follow-up without images",
timestamp: "2026-04-20T10:01:00Z",
},
],
});
const { result } = renderHook(() => useSessionHistory("websocket:chat-media"), {
wrapper: wrap(fakeClient()),
});
await waitFor(() => expect(result.current.loading).toBe(false));
const [first, second, third] = result.current.messages;
expect(first.role).toBe("user");
expect(first.images).toEqual([
{ url: "/api/media/sig-1/payload-1", name: "snap.png" },
{ url: "/api/media/sig-2/payload-2", name: "diag.jpg" },
]);
expect(second.role).toBe("assistant");
expect(second.images).toBeUndefined();
expect(third.role).toBe("user");
expect(third.images).toBeUndefined();
});
it("keeps the session in the list when delete fails", async () => {
vi.mocked(api.listSessions).mockResolvedValue([
{
+264
View File
@@ -0,0 +1,264 @@
/**
* Off-main-thread image encoder.
*
* Accepts a ``File``, validates it via magic bytes (ignoring the extension to
* defeat rename-based spoofs), and either passes through or *normalizes* the
* bytes so the resulting base64 data URL stays ≤ ``TARGET_MAX_BYTES``. The
* normalization path uses ``createImageBitmap`` + ``OffscreenCanvas`` so the
* full decode/resize/re-encode cycle never blocks the UI thread.
*
* Output contract:
* ``{ok: true, dataUrl, mime, bytes, origBytes, normalized}`` on success, or
* ``{ok: false, reason}`` for every recoverable failure — magic-bytes
* mismatch, unsupported MIME, decode error, or a post-normalization payload
* that *still* exceeds the budget (extreme aspect ratios).
*/
/// <reference lib="webworker" />
// --- Types -------------------------------------------------------------------
export type EncodeInput = {
id: string;
file: File;
};
export type EncodeSuccess = {
id: string;
ok: true;
dataUrl: string;
mime: string;
bytes: number;
origBytes: number;
/** True iff the Worker re-encoded the image to hit the size budget. */
normalized: boolean;
};
export type EncodeFailure = {
id: string;
ok: false;
reason:
| "invalid_mime"
| "magic_mismatch"
| "too_large_after_normalize"
| "decode_failed"
| "io";
};
export type EncodeResponse = EncodeSuccess | EncodeFailure;
// --- Budgets -----------------------------------------------------------------
/** Upper bound for the final base64-decoded payload. Matches the server-side
* safeguard (8 MB) minus safety margin; anything this function yields should
* safely pass ``_MAX_IMAGE_BYTES`` on the server. */
export const TARGET_MAX_BYTES = 6 * 1024 * 1024;
/** Long-edge pixel cap when we resize a large image. 2048 keeps retina UIs
* crisp while bounding decode cost and matching most LLM vision tiers'
* internal downscale target. */
const NORMALIZE_MAX_EDGE = 2048;
/** JPEG/WebP quality during normalization. 0.85 is the sweet spot — visually
* lossless for content photography, ~30% smaller than libjpeg default. */
const WEBP_QUALITY = 0.85;
/** PNG / GIF kept as PNG after normalization so crisp UI screenshots stay
* lossless. JPEG / WebP re-encode as WebP for better compression. */
const NORMALIZE_LOSSY_MIMES = new Set(["image/jpeg", "image/webp"]);
const SUPPORTED_MIMES = new Set([
"image/png",
"image/jpeg",
"image/webp",
"image/gif",
]);
// --- Magic bytes -------------------------------------------------------------
/** Sniff the first 12 bytes; returns the canonical MIME or ``null``.
*
* Covers PNG, JPEG, WebP, GIF — the same whitelist honoured by the server.
*/
export function sniffImageMime(bytes: Uint8Array): string | null {
if (bytes.length >= 8) {
if (
bytes[0] === 0x89 &&
bytes[1] === 0x50 &&
bytes[2] === 0x4e &&
bytes[3] === 0x47 &&
bytes[4] === 0x0d &&
bytes[5] === 0x0a &&
bytes[6] === 0x1a &&
bytes[7] === 0x0a
) {
return "image/png";
}
}
if (bytes.length >= 3) {
if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
return "image/jpeg";
}
}
if (bytes.length >= 6) {
const g1 =
bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46 &&
bytes[3] === 0x38 && bytes[5] === 0x61;
if (g1 && (bytes[4] === 0x37 || bytes[4] === 0x39)) {
return "image/gif";
}
}
if (bytes.length >= 12) {
const riff =
bytes[0] === 0x52 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x46;
const webp =
bytes[8] === 0x57 && bytes[9] === 0x45 && bytes[10] === 0x42 && bytes[11] === 0x50;
if (riff && webp) return "image/webp";
}
return null;
}
// --- Encoder -----------------------------------------------------------------
function bufferToBase64(buf: ArrayBuffer): string {
// ``btoa`` can't take large strings — chunk through 32 KB windows.
const bytes = new Uint8Array(buf);
let binary = "";
const CHUNK = 0x8000;
for (let i = 0; i < bytes.length; i += CHUNK) {
binary += String.fromCharCode.apply(
null,
bytes.subarray(i, i + CHUNK) as unknown as number[],
);
}
return self.btoa(binary);
}
function computeScaledDims(
srcW: number,
srcH: number,
maxEdge: number,
): { w: number; h: number } {
const longest = Math.max(srcW, srcH);
if (longest <= maxEdge) return { w: srcW, h: srcH };
const scale = maxEdge / longest;
return {
w: Math.max(1, Math.round(srcW * scale)),
h: Math.max(1, Math.round(srcH * scale)),
};
}
async function normalize(
file: File,
sourceMime: string,
): Promise<{ dataUrl: string; mime: string; bytes: number } | { error: EncodeFailure["reason"] }> {
// Re-encode paths: JPEG/WebP → WebP q=0.85; PNG/GIF → PNG (keep crisp).
const targetMime = NORMALIZE_LOSSY_MIMES.has(sourceMime)
? "image/webp"
: "image/png";
let bitmap: ImageBitmap;
try {
bitmap = await createImageBitmap(file);
} catch {
return { error: "decode_failed" };
}
const { w, h } = computeScaledDims(bitmap.width, bitmap.height, NORMALIZE_MAX_EDGE);
try {
const canvas = new OffscreenCanvas(w, h);
const ctx = canvas.getContext("2d", { alpha: true });
if (!ctx) {
bitmap.close();
return { error: "decode_failed" };
}
ctx.imageSmoothingQuality = "high";
ctx.drawImage(bitmap, 0, 0, w, h);
bitmap.close();
const options: ImageEncodeOptions = { type: targetMime };
if (targetMime === "image/webp") options.quality = WEBP_QUALITY;
const blob = await canvas.convertToBlob(options);
if (blob.size > TARGET_MAX_BYTES) {
return { error: "too_large_after_normalize" };
}
const buf = await blob.arrayBuffer();
const dataUrl = `data:${targetMime};base64,${bufferToBase64(buf)}`;
return { dataUrl, mime: targetMime, bytes: blob.size };
} catch {
try {
bitmap.close();
} catch {
// bitmap already closed
}
return { error: "decode_failed" };
}
}
export async function encodeImageInWorker(
input: EncodeInput,
): Promise<EncodeResponse> {
const { id, file } = input;
const origBytes = file.size;
let buffer: ArrayBuffer;
try {
buffer = await file.arrayBuffer();
} catch {
return { id, ok: false, reason: "io" };
}
const head = new Uint8Array(buffer.slice(0, 12));
const sniffed = sniffImageMime(head);
if (!sniffed) return { id, ok: false, reason: "magic_mismatch" };
if (!SUPPORTED_MIMES.has(sniffed)) {
return { id, ok: false, reason: "invalid_mime" };
}
// Defend against MIME spoofing: the declared ``file.type`` can lie.
if (file.type && SUPPORTED_MIMES.has(file.type) && file.type !== sniffed) {
// Trust the magic bytes; proceed with the sniffed MIME.
}
if (origBytes <= TARGET_MAX_BYTES) {
const dataUrl = `data:${sniffed};base64,${bufferToBase64(buffer)}`;
return {
id,
ok: true,
dataUrl,
mime: sniffed,
bytes: origBytes,
origBytes,
normalized: false,
};
}
const result = await normalize(file, sniffed);
if ("error" in result) {
return { id, ok: false, reason: result.error };
}
return {
id,
ok: true,
dataUrl: result.dataUrl,
mime: result.mime,
bytes: result.bytes,
origBytes,
normalized: true,
};
}
// --- Worker boot -------------------------------------------------------------
// Only attach the message listener when running *inside* a Worker so the same
// module can be imported by tests (and by the thin ``imageEncode.ts`` wrapper
// in the main thread, which also calls ``encodeImageInWorker`` as a
// fall-through path when the Worker isn't available).
declare const self: DedicatedWorkerGlobalScope;
if (
typeof self !== "undefined" &&
typeof (self as unknown as { importScripts?: unknown }).importScripts ===
"function"
) {
self.addEventListener("message", async (event: MessageEvent<EncodeInput>) => {
const response = await encodeImageInWorker(event.data);
self.postMessage(response);
});
}