Grayscale/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(webui): render video media attachments

Browse Source 
Add signed media URLs to live WebSocket replies and teach the WebUI to classify and render video attachments, so bot-sent videos can play inline in both live chats and session history.

Made-with: Cursor
This commit is contained in:
Xubin Ren
2026-04-25 03:20:40 +08:00
committed by Xubin Ren
parent be05189f39
commit e52fe2a8e2
10 changed files with 327 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webui/src/components/MessageBubble.tsx
+92 -5
View File
@@ -1,11 +1,11 @@
import { useState } from "react";
import { ChevronRight, ImageIcon, Wrench } from "lucide-react";
import { ChevronRight, FileIcon, ImageIcon, PlaySquare, Wrench } from "lucide-react";
import { useTranslation } from "react-i18next";
import { ImageLightbox } from "@/components/ImageLightbox";
import { MarkdownText } from "@/components/MarkdownText";
import { cn } from "@/lib/utils";
import type { UIImage, UIMessage } from "@/lib/types";
import type { UIImage, UIMediaAttachment, UIMessage } from "@/lib/types";
interface MessageBubbleProps {
message: UIMessage;
@@ -29,7 +29,9 @@ export function MessageBubble({ message }: MessageBubbleProps) {
if (message.role === "user") {
const images = message.images ?? [];
const media = message.media ?? [];
const hasImages = images.length > 0;
const hasMedia = media.length > 0;
const hasText = message.content.trim().length > 0;
return (
<div
@@ -38,7 +40,10 @@ export function MessageBubble({ message }: MessageBubbleProps) {
baseAnim,
)}
>
{hasImages ? <UserImages images={images} /> : null}
{hasImages ? <UserImages images={images} align="right" /> : null}
{!hasImages && hasMedia ? (
<MessageMedia media={media} align="right" />
) : null}
{hasText ? (
<p
className={cn(
@@ -54,6 +59,7 @@ export function MessageBubble({ message }: MessageBubbleProps) {
}
const empty = message.content.trim().length === 0;
const media = message.media ?? [];
return (
<div className={cn("w-full text-sm", baseAnim)} style={{ lineHeight: "var(--cjk-line-height)" }}>
{empty && message.isStreaming ? (
@@ -62,12 +68,82 @@ export function MessageBubble({ message }: MessageBubbleProps) {
<>
<MarkdownText>{message.content}</MarkdownText>
{message.isStreaming && <StreamCursor />}
{media.length > 0 ? <MessageMedia media={media} align="left" /> : null}
</>
)}
</div>
);
}
function MessageMedia({
media,
align,
}: {
media: UIMediaAttachment[];
align: "left" | "right";
}) {
if (media.length === 0) return null;
const images = media
.filter((item) => item.kind === "image")
.map(({ url, name }) => ({ url, name }));
const nonImages = media.filter((item) => item.kind !== "image");
return (
<div
className={cn(
"mt-2 flex flex-wrap gap-2",
align === "right" ? "justify-end" : "justify-start",
)}
>
{images.length > 0 ? <UserImages images={images} align={align} /> : null}
{nonImages.map((item, i) => (
<MediaCell key={`${item.url ?? item.name ?? item.kind}-${i}`} media={item} />
))}
</div>
);
}
function MediaCell({ media }: { media: UIMediaAttachment }) {
const { t } = useTranslation();
const hasUrl = typeof media.url === "string" && media.url.length > 0;
if (media.kind === "video" && hasUrl) {
return (
<figure className="max-w-[min(100%,32rem)] overflow-hidden rounded-[14px] border border-border/60 bg-muted/40">
<video
src={media.url}
controls
preload="metadata"
className="block max-h-[26rem] w-full bg-black"
aria-label={media.name ? `${t("message.videoAttachment", { defaultValue: "Video attachment" })}: ${media.name}` : t("message.videoAttachment", { defaultValue: "Video attachment" })}
/>
{media.name ? (
<figcaption className="truncate px-3 py-1.5 text-[11.5px] text-muted-foreground">
{media.name}
</figcaption>
) : null}
</figure>
);
}
const label =
media.kind === "video"
? t("message.videoAttachment", { defaultValue: "Video attachment" })
: t("message.fileAttachment", { defaultValue: "File attachment" });
const Icon = media.kind === "video" ? PlaySquare : FileIcon;
return (
<div
className="flex max-w-[18rem] items-center gap-2 rounded-[14px] border border-border/60 bg-muted/40 px-3 py-2 text-xs text-muted-foreground"
title={media.name ?? undefined}
aria-label={label}
>
<Icon className="h-4 w-4 flex-none" aria-hidden />
<span className="truncate">{media.name ?? label}</span>
</div>
);
}
/**
* Right-aligned preview row for images attached to a user turn.
*
@@ -82,7 +158,13 @@ export function MessageBubble({ message }: MessageBubbleProps) {
* have no URL (the backend strips data URLs before persisting), so we
* render a labelled placeholder tile instead of a broken ``<img>``.
*/
function UserImages({ images }: { images: UIImage[] }) {
function UserImages({
images,
align = "right",
}: {
images: UIImage[];
align?: "left" | "right";
}) {
const { t } = useTranslation();
// Only real-URL images can open in the lightbox; historical-replay
// placeholders (no URL) have nothing to zoom into.
@@ -98,7 +180,12 @@ function UserImages({ images }: { images: UIImage[] }) {
return (
<>
<div className="ml-auto flex flex-wrap items-end justify-end gap-2">
<div
className={cn(
"flex flex-wrap items-end gap-2",
align === "right" ? "ml-auto justify-end" : "mr-auto justify-start",
)}
>
{images.map((img, i) => (
<UserImageCell
key={`${img.url ?? "placeholder"}-${i}`}
webui/src/hooks/useNanobotStream.ts
+6
View File
@@ -1,6 +1,7 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { useClient } from "@/providers/ClientProvider";
import { toMediaAttachment } from "@/lib/media";
import type { StreamError } from "@/lib/nanobot-client";
import type {
InboundEvent,
@@ -148,6 +149,10 @@ export function useNanobotStream(
return;
}
const media = ev.media_urls?.length
? ev.media_urls.map((m) => toMediaAttachment(m))
: ev.media?.map((url) => toMediaAttachment({ url }));
// A complete (non-streamed) assistant message. If a stream was in
// flight, drop the placeholder so we don't render the text twice.
const activeId = buffer.current?.messageId;
@@ -162,6 +167,7 @@ export function useNanobotStream(
role: "assistant",
content: ev.text,
createdAt: Date.now(),
...(media && media.length > 0 ? { media } : {}),
},
];
});
webui/src/hooks/useSessions.ts
+11 -10
View File
@@ -9,6 +9,7 @@ import {
listSessions,
} from "@/lib/api";
import { deriveTitle } from "@/lib/format";
import { toMediaAttachment } from "@/lib/media";
import type { ChatSummary, UIMessage } from "@/lib/types";
const EMPTY_MESSAGES: UIMessage[] = [];
@@ -123,17 +124,16 @@ export function useSessionHistory(key: string | null): {
const ui: UIMessage[] = body.messages.flatMap((m, idx) => {
if (m.role !== "user" && m.role !== "assistant") return [];
if (typeof m.content !== "string") return [];
// Hydrate signed media URLs into the bubble's ``images`` slot so
// historical user turns render real previews (the live-send path
// uses data URLs; both shapes converge on the same ``UIImage``).
// Hydrate signed media URLs into generic UI attachments. Image-only
// user turns still populate the legacy ``images`` slot so the
// existing optimistic-send and lightbox paths remain unchanged.
const media =
Array.isArray(m.media_urls) && m.media_urls.length > 0
? m.media_urls.map((mu) => toMediaAttachment(mu))
: undefined;
const images =
m.role === "user" &&
Array.isArray(m.media_urls) &&
m.media_urls.length > 0
? m.media_urls.map((mu) => ({
url: mu.url,
name: mu.name,
}))
m.role === "user" && media?.every((item) => item.kind === "image")
? media.map((item) => ({ url: item.url, name: item.name }))
: undefined;
return [
{
@@ -142,6 +142,7 @@ export function useSessionHistory(key: string | null): {
content: m.content,
createdAt: m.timestamp ? Date.parse(m.timestamp) : Date.now(),
...(images ? { images } : {}),
...(media ? { media } : {}),
},
];
});
webui/src/lib/media.ts
+59
View File
@@ -0,0 +1,59 @@
import type { UIMediaAttachment, UIMediaKind } from "@/lib/types";
const IMAGE_EXTENSIONS = new Set([
".png",
".jpg",
".jpeg",
".gif",
".webp",
".bmp",
".ico",
".tif",
".tiff",
]);
const VIDEO_EXTENSIONS = new Set([
".mp4",
".webm",
".mov",
".m4v",
".avi",
".mkv",
".3gp",
]);
function cleanPath(value: string): string {
return value.split(/[?#]/, 1)[0]?.toLowerCase() ?? "";
}
function extensionOf(value?: string): string {
if (!value) return "";
const path = cleanPath(value);
const dot = path.lastIndexOf(".");
if (dot < 0) return "";
return path.slice(dot);
}
export function inferMediaKind(media: { url?: string; name?: string }): UIMediaKind {
const url = media.url ?? "";
if (url.startsWith("data:image/")) return "image";
if (url.startsWith("data:video/")) return "video";
const ext = extensionOf(media.name) || extensionOf(url);
if (IMAGE_EXTENSIONS.has(ext)) return "image";
if (VIDEO_EXTENSIONS.has(ext)) return "video";
return "file";
}
export function toMediaAttachment(media: {
url?: string;
name?: string;
kind?: UIMediaKind;
}): UIMediaAttachment {
return {
kind: media.kind ?? inferMediaKind(media),
url: media.url,
name: media.name,
};
}
webui/src/lib/types.ts
+11
View File
@@ -22,6 +22,14 @@ export interface UIImage {
name?: string;
}
export type UIMediaKind = "image" | "video" | "file";
export interface UIMediaAttachment {
kind: UIMediaKind;
url?: string;
name?: string;
}
export interface UIMessage {
id: string;
role: Role;
@@ -34,6 +42,8 @@ export interface UIMessage {
traces?: string[];
/** User turn: optimistic blob URLs for preview. Replay: placeholder chips. */
images?: UIImage[];
/** Signed or local UI-renderable media attachments. */
media?: UIMediaAttachment[];
}
export interface ChatSummary {
@@ -71,6 +81,7 @@ export type InboundEvent =
text: string;
reply_to?: string;
media?: string[];
media_urls?: Array<{ url: string; name?: string }>;
/** Present when the frame is an agent breadcrumb (e.g. tool hint,
* generic progress line) rather than a conversational reply. */
kind?: "tool_hint" | "progress";
webui/src/tests/message-bubble.test.tsx
+24
View File
@@ -40,4 +40,28 @@ describe("MessageBubble", () => {
fireEvent.click(toggle);
expect(screen.queryByText('weather("get")')).not.toBeInTheDocument();
});
it("renders video media as an inline player", () => {
const message: UIMessage = {
id: "a1",
role: "assistant",
content: "here is the clip",
createdAt: Date.now(),
media: [
{
kind: "video",
url: "/api/media/sig/payload",
name: "demo.mp4",
},
],
};
const { container } = render(<MessageBubble message={message} />);
expect(screen.getByText("here is the clip")).toBeInTheDocument();
const video = screen.getByLabelText(/video attachment/i);
expect(video.tagName).toBe("VIDEO");
expect(video).toHaveAttribute("src", "/api/media/sig/payload");
expect(container.querySelector("video[controls]")).toBeInTheDocument();
});
});
webui/src/tests/useNanobotStream.test.tsx
+21
View File
@@ -92,4 +92,25 @@ describe("useNanobotStream", () => {
expect(result.current.messages[1].role).toBe("assistant");
expect(result.current.messages[1].kind).toBeUndefined();
});
it("attaches assistant media_urls to complete messages", () => {
const fake = fakeClient();
const { result } = renderHook(() => useNanobotStream("chat-m", []), {
wrapper: wrap(fake.client),
});
act(() => {
fake.emit("chat-m", {
event: "message",
chat_id: "chat-m",
text: "video ready",
media_urls: [{ url: "/api/media/sig/payload", name: "demo.mp4" }],
});
});
expect(result.current.messages).toHaveLength(1);
expect(result.current.messages[0].media).toEqual([
{ kind: "video", url: "/api/media/sig/payload", name: "demo.mp4" },
]);
});
});
webui/src/tests/useSessions.test.tsx
+34
View File
@@ -130,12 +130,46 @@ describe("useSessions", () => {
{ url: "/api/media/sig-1/payload-1", name: "snap.png" },
{ url: "/api/media/sig-2/payload-2", name: "diag.jpg" },
]);
expect(first.media).toEqual([
{ kind: "image", url: "/api/media/sig-1/payload-1", name: "snap.png" },
{ kind: "image", url: "/api/media/sig-2/payload-2", name: "diag.jpg" },
]);
expect(second.role).toBe("assistant");
expect(second.images).toBeUndefined();
expect(third.role).toBe("user");
expect(third.images).toBeUndefined();
});
it("hydrates historical assistant video media_urls into media attachments", async () => {
vi.mocked(api.fetchSessionMessages).mockResolvedValue({
key: "websocket:chat-video",
created_at: "2026-04-20T10:00:00Z",
updated_at: "2026-04-20T10:05:00Z",
messages: [
{
role: "assistant",
content: "clip ready",
timestamp: "2026-04-20T10:00:01Z",
media_urls: [
{ url: "/api/media/sig-v/payload-v", name: "clip.mp4" },
],
},
],
});
const { result } = renderHook(() => useSessionHistory("websocket:chat-video"), {
wrapper: wrap(fakeClient()),
});
await waitFor(() => expect(result.current.loading).toBe(false));
expect(result.current.messages[0].role).toBe("assistant");
expect(result.current.messages[0].images).toBeUndefined();
expect(result.current.messages[0].media).toEqual([
{ kind: "video", url: "/api/media/sig-v/payload-v", name: "clip.mp4" },
]);
});
it("keeps the session in the list when delete fails", async () => {
vi.mocked(api.listSessions).mockResolvedValue([
{