323 lines
11 KiB
TypeScript
323 lines
11 KiB
TypeScript
import { toMediaAttachment } from "@/lib/media";
|
|
import type { ToolProgressEvent, UIMediaAttachment, UIMessage } from "@/lib/types";
|
|
|
|
export type ActivityItemType = "reasoning" | "tool" | "cli" | "mcp" | "file_edit" | "media";
|
|
export type ActivityStepStatus = "pending" | "running" | "done" | "error";
|
|
export type ActivityStepSource = "reasoning" | "tool" | "web" | "browser" | "shell" | "mcp" | "file" | "media";
|
|
|
|
export interface ActivityItem {
|
|
type: ActivityItemType;
|
|
message: UIMessage;
|
|
}
|
|
|
|
export interface ActivityEvidence {
|
|
id: string;
|
|
attachment: UIMediaAttachment;
|
|
caption?: string;
|
|
source: ActivityStepSource;
|
|
}
|
|
|
|
export interface ActivityStepItem {
|
|
id: string;
|
|
label: string;
|
|
detail?: string;
|
|
status: ActivityStepStatus;
|
|
source: ActivityStepSource;
|
|
preview?: ActivityEvidence[];
|
|
error?: string;
|
|
}
|
|
|
|
export interface ActivityGroup {
|
|
id: string;
|
|
title: string;
|
|
source: ActivityStepSource;
|
|
steps: ActivityStepItem[];
|
|
}
|
|
|
|
export type TurnUnit =
|
|
| { type: "activity"; messages: UIMessage[]; items: ActivityItem[]; turnLatencyMs?: number }
|
|
| { type: "message"; message: UIMessage };
|
|
|
|
export function isReasoningOnlyAssistant(message: UIMessage): boolean {
|
|
if (message.role !== "assistant" || message.kind === "trace") return false;
|
|
if (message.content.trim().length > 0) return false;
|
|
return !!(message.reasoning?.length || message.reasoningStreaming || message.isStreaming);
|
|
}
|
|
|
|
export function isAgentActivityMember(message: UIMessage): boolean {
|
|
return isReasoningOnlyAssistant(message) || message.kind === "trace";
|
|
}
|
|
|
|
export function normalizeActivityTimeline(messages: UIMessage[]): TurnUnit[] {
|
|
const units: TurnUnit[] = [];
|
|
let turnMessages: UIMessage[] = [];
|
|
|
|
const flushTurn = () => {
|
|
if (turnMessages.length === 0) return;
|
|
|
|
const visibleMessages = visibleMessagesForTurn(turnMessages);
|
|
let visibleIndex = 0;
|
|
let activityMessages: UIMessage[] = [];
|
|
|
|
const flushActivityMessages = () => {
|
|
if (!activityMessages.length) return;
|
|
pushActivityUnits(units, activityMessages, visibleMessages.slice(visibleIndex));
|
|
activityMessages = [];
|
|
};
|
|
|
|
for (const message of turnMessages) {
|
|
if (isAgentActivityMember(message)) {
|
|
activityMessages.push(message);
|
|
continue;
|
|
}
|
|
|
|
if (assistantHasInlineReasoning(message)) {
|
|
activityMessages.push(reasoningOnlyMessageFromAnswer(message));
|
|
flushActivityMessages();
|
|
units.push({ type: "message", message: stripInlineReasoning(message) });
|
|
visibleIndex += 1;
|
|
continue;
|
|
}
|
|
|
|
flushActivityMessages();
|
|
units.push({ type: "message", message });
|
|
visibleIndex += 1;
|
|
}
|
|
|
|
flushActivityMessages();
|
|
turnMessages = [];
|
|
};
|
|
|
|
for (const message of messages) {
|
|
if (message.role === "user") {
|
|
flushTurn();
|
|
units.push({ type: "message", message });
|
|
continue;
|
|
}
|
|
|
|
turnMessages.push(message);
|
|
}
|
|
|
|
flushTurn();
|
|
return units;
|
|
}
|
|
|
|
function visibleMessagesForTurn(messages: UIMessage[]): UIMessage[] {
|
|
const visibleMessages: UIMessage[] = [];
|
|
for (const message of messages) {
|
|
if (isAgentActivityMember(message)) continue;
|
|
visibleMessages.push(assistantHasInlineReasoning(message) ? stripInlineReasoning(message) : message);
|
|
}
|
|
return visibleMessages;
|
|
}
|
|
|
|
function pushActivityUnits(units: TurnUnit[], activityMessages: UIMessage[], visibleMessages: UIMessage[]) {
|
|
let runMessages: UIMessage[] = [];
|
|
let runBucket: "file" | "other" | undefined;
|
|
let runSegmentId: string | undefined;
|
|
|
|
const flushRun = () => {
|
|
if (!runMessages.length) return;
|
|
units.push({
|
|
type: "activity",
|
|
messages: runMessages,
|
|
items: runMessages.flatMap(activityItemsForMessage),
|
|
turnLatencyMs: activityTurnLatencyMs(runMessages, visibleMessages),
|
|
});
|
|
runMessages = [];
|
|
runBucket = undefined;
|
|
runSegmentId = undefined;
|
|
};
|
|
|
|
for (const message of activityMessages) {
|
|
const bucket = isFileEditActivityMessage(message) ? "file" : "other";
|
|
const segmentId = message.activitySegmentId;
|
|
const segmentChanged =
|
|
bucket === "file"
|
|
&& runBucket === "file"
|
|
&& !!runSegmentId
|
|
&& !!segmentId
|
|
&& runSegmentId !== segmentId;
|
|
if ((runBucket && bucket !== runBucket) || segmentChanged) {
|
|
flushRun();
|
|
}
|
|
runBucket = bucket;
|
|
if (segmentId) runSegmentId = segmentId;
|
|
runMessages.push(message);
|
|
}
|
|
|
|
flushRun();
|
|
}
|
|
|
|
function isFileEditActivityMessage(message: UIMessage): boolean {
|
|
return message.kind === "trace" && !!message.fileEdits?.length;
|
|
}
|
|
|
|
function assistantHasInlineReasoning(message: UIMessage): boolean {
|
|
return (
|
|
message.role === "assistant"
|
|
&& message.kind !== "trace"
|
|
&& message.content.trim().length > 0
|
|
&& (!!message.reasoning?.trim() || !!message.reasoningStreaming)
|
|
);
|
|
}
|
|
|
|
function reasoningOnlyMessageFromAnswer(message: UIMessage): UIMessage {
|
|
return {
|
|
id: `${message.id}-reasoning`,
|
|
role: "assistant",
|
|
content: "",
|
|
createdAt: message.createdAt,
|
|
reasoning: message.reasoning,
|
|
reasoningStreaming: message.reasoningStreaming,
|
|
isStreaming: message.reasoningStreaming,
|
|
activitySegmentId: message.activitySegmentId,
|
|
latencyMs: message.latencyMs,
|
|
};
|
|
}
|
|
|
|
function stripInlineReasoning(message: UIMessage): UIMessage {
|
|
const next = { ...message };
|
|
delete next.reasoning;
|
|
delete next.reasoningStreaming;
|
|
return next;
|
|
}
|
|
|
|
function activityItemsForMessage(message: UIMessage): ActivityItem[] {
|
|
if (isReasoningOnlyAssistant(message)) {
|
|
return [{ type: "reasoning", message }];
|
|
}
|
|
if (message.kind !== "trace") return [];
|
|
|
|
const items: ActivityItem[] = [];
|
|
if (message.fileEdits?.length) {
|
|
items.push({ type: "file_edit", message });
|
|
}
|
|
for (const event of message.toolEvents ?? []) {
|
|
const name = String(event.name ?? "").toLowerCase();
|
|
if (name === "run_cli_app") {
|
|
items.push({ type: "cli", message });
|
|
} else if (name === "mcp") {
|
|
items.push({ type: "mcp", message });
|
|
} else {
|
|
items.push({ type: "tool", message });
|
|
}
|
|
}
|
|
if (items.length === 0 && (message.traces?.length || message.content.trim())) {
|
|
items.push({ type: "tool", message });
|
|
}
|
|
if (message.media?.length) {
|
|
items.push({ type: "media", message });
|
|
}
|
|
return items;
|
|
}
|
|
|
|
function activityTurnLatencyMs(activityMessages: UIMessage[], visibleMessages: UIMessage[]): number | undefined {
|
|
for (let i = activityMessages.length - 1; i >= 0; i -= 1) {
|
|
const latency = activityMessages[i].latencyMs;
|
|
if (isValidLatency(latency)) return latency;
|
|
}
|
|
for (let i = visibleMessages.length - 1; i >= 0; i -= 1) {
|
|
const latency = visibleMessages[i].latencyMs;
|
|
if (isValidLatency(latency)) return latency;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function isValidLatency(value: unknown): value is number {
|
|
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
}
|
|
|
|
export function activityEvidenceFromToolEvent(event: ToolProgressEvent): ActivityEvidence[] {
|
|
const source = activitySourceFromToolName(toolEventName(event));
|
|
const evidence: ActivityEvidence[] = [];
|
|
const extras = [
|
|
...unknownList((event as { embeds?: unknown }).embeds),
|
|
...unknownList((event as { files?: unknown }).files),
|
|
];
|
|
extras.forEach((value, index) => {
|
|
const attachment = mediaAttachmentFromUnknown(value);
|
|
if (!attachment) return;
|
|
evidence.push({
|
|
id: `${event.call_id || toolEventName(event) || "tool"}:${index}:${attachment.url || attachment.name || attachment.kind}`,
|
|
attachment,
|
|
caption: attachment.name,
|
|
source,
|
|
});
|
|
});
|
|
return evidence;
|
|
}
|
|
|
|
export function activityEvidenceFromMessageMedia(message: UIMessage): ActivityEvidence[] {
|
|
return (message.media ?? []).map((attachment, index) => ({
|
|
id: `${message.id}:media:${index}:${attachment.url || attachment.name || attachment.kind}`,
|
|
attachment,
|
|
caption: attachment.name,
|
|
source: "media",
|
|
}));
|
|
}
|
|
|
|
function unknownList(value: unknown): unknown[] {
|
|
return Array.isArray(value) ? value : [];
|
|
}
|
|
|
|
function toolEventName(event: ToolProgressEvent): string {
|
|
return typeof (event as { function?: { name?: unknown } }).function?.name === "string"
|
|
? String((event as { function?: { name?: unknown } }).function?.name)
|
|
: typeof event.name === "string"
|
|
? event.name
|
|
: "";
|
|
}
|
|
|
|
function activitySourceFromToolName(name: string): ActivityStepSource {
|
|
const compact = name.toLowerCase();
|
|
if (compact.includes("browser") || compact.includes("screenshot")) return "browser";
|
|
if (compact.includes("web") || compact.includes("search") || compact.includes("fetch") || compact.includes("read")) return "web";
|
|
if (compact.includes("exec") || compact.includes("shell") || compact.includes("cli")) return "shell";
|
|
if (compact.startsWith("mcp_") || compact === "mcp") return "mcp";
|
|
if (compact.includes("file") || compact.includes("patch")) return "file";
|
|
if (compact.includes("image") || compact.includes("video") || compact.includes("media")) return "media";
|
|
return "tool";
|
|
}
|
|
|
|
function mediaAttachmentFromUnknown(value: unknown): UIMediaAttachment | null {
|
|
if (typeof value === "string") {
|
|
const text = value.trim();
|
|
if (!text) return null;
|
|
return toMediaAttachment({ url: looksLikeUrl(text) ? text : undefined, name: baseName(text) });
|
|
}
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
const record = value as Record<string, unknown>;
|
|
const url = stringField(record, ["url", "href", "src", "uri", "signed_url", "thumbnail_url"]);
|
|
const path = stringField(record, ["path", "absolute_path", "file", "filename"]);
|
|
const name = stringField(record, ["name", "filename", "title", "label"]) ?? baseName(url ?? path ?? "");
|
|
const kind = mediaKindFromRecord(record, url, name);
|
|
return toMediaAttachment({ url, name, kind });
|
|
}
|
|
|
|
function stringField(record: Record<string, unknown>, keys: string[]): string | undefined {
|
|
for (const key of keys) {
|
|
const value = record[key];
|
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function mediaKindFromRecord(record: Record<string, unknown>, url?: string, name?: string): UIMediaAttachment["kind"] | undefined {
|
|
const raw = stringField(record, ["kind", "type", "mime", "mime_type", "content_type"])?.toLowerCase() ?? "";
|
|
if (raw.includes("image") || raw.includes("screenshot")) return "image";
|
|
if (raw.includes("video") || raw.includes("mp4") || raw.includes("quicktime")) return "video";
|
|
if (raw.includes("file") || raw.includes("document")) return "file";
|
|
return toMediaAttachment({ url, name }).kind;
|
|
}
|
|
|
|
function looksLikeUrl(value: string): boolean {
|
|
return /^(https?:|data:|\/api\/|blob:)/i.test(value);
|
|
}
|
|
|
|
function baseName(value: string): string | undefined {
|
|
const clean = value.split(/[?#]/, 1)[0] ?? "";
|
|
const last = clean.split(/[\\/]/).filter(Boolean).pop();
|
|
return last || undefined;
|
|
}
|