feat(webui): refine output timeline and model controls (#4108)

* feat(webui): refine output timeline and composer queue

* feat(webui): add provider model picker

* fix(webui): polish model settings and heartbeat checks

* chore: keep heartbeat changes out of webui pr

* refactor(webui): isolate settings routes

* fix(providers): align minimax anthropic test

* fix(providers): keep minimax anthropic base sdk-compatible

* fix(providers): normalize anthropic base urls

webui/src/lib/activity-timeline.ts

@@ -0,0 +1,297 @@
+import { toMediaAttachment } from "@/lib/media";
+import type { ToolProgressEvent, UIMediaAttachment, UIMessage } from "@/lib/types";
+
+export type ActivityItemType = "reasoning" | "tool" | "cli" | "mcp" | "file_edit" | "media";
+export type ActivityStepStatus = "pending" | "running" | "done" | "error";
+export type ActivityStepSource = "reasoning" | "tool" | "web" | "browser" | "shell" | "mcp" | "file" | "media";
+
+export interface ActivityItem {
+  type: ActivityItemType;
+  message: UIMessage;
+}
+
+export interface ActivityEvidence {
+  id: string;
+  attachment: UIMediaAttachment;
+  caption?: string;
+  source: ActivityStepSource;
+}
+
+export interface ActivityStepItem {
+  id: string;
+  label: string;
+  detail?: string;
+  status: ActivityStepStatus;
+  source: ActivityStepSource;
+  preview?: ActivityEvidence[];
+  error?: string;
+}
+
+export interface ActivityGroup {
+  id: string;
+  title: string;
+  source: ActivityStepSource;
+  steps: ActivityStepItem[];
+}
+
+export type TurnUnit =
+  | { type: "activity"; messages: UIMessage[]; items: ActivityItem[]; turnLatencyMs?: number }
+  | { type: "message"; message: UIMessage };
+
+export function isReasoningOnlyAssistant(message: UIMessage): boolean {
+  if (message.role !== "assistant" || message.kind === "trace") return false;
+  if (message.content.trim().length > 0) return false;
+  return !!(message.reasoning?.length || message.reasoningStreaming || message.isStreaming);
+}
+
+export function isAgentActivityMember(message: UIMessage): boolean {
+  return isReasoningOnlyAssistant(message) || message.kind === "trace";
+}
+
+export function normalizeActivityTimeline(messages: UIMessage[]): TurnUnit[] {
+  const units: TurnUnit[] = [];
+  let turnMessages: UIMessage[] = [];
+
+  const flushTurn = () => {
+    if (turnMessages.length === 0) return;
+
+    const activityMessages: UIMessage[] = [];
+    const visibleMessages: UIMessage[] = [];
+
+    for (const message of turnMessages) {
+      if (isAgentActivityMember(message)) {
+        activityMessages.push(message);
+        continue;
+      }
+
+      if (assistantHasInlineReasoning(message)) {
+        activityMessages.push(reasoningOnlyMessageFromAnswer(message));
+        visibleMessages.push(stripInlineReasoning(message));
+        continue;
+      }
+
+      visibleMessages.push(message);
+    }
+
+    pushActivityUnits(units, activityMessages, visibleMessages);
+
+    for (const message of visibleMessages) {
+      units.push({ type: "message", message });
+    }
+
+    turnMessages = [];
+  };
+
+  for (const message of messages) {
+    if (message.role === "user") {
+      flushTurn();
+      units.push({ type: "message", message });
+      continue;
+    }
+
+    turnMessages.push(message);
+  }
+
+  flushTurn();
+  return units;
+}
+
+function pushActivityUnits(units: TurnUnit[], activityMessages: UIMessage[], visibleMessages: UIMessage[]) {
+  let runMessages: UIMessage[] = [];
+  let runBucket: "file" | "other" | undefined;
+
+  const flushRun = () => {
+    if (!runMessages.length) return;
+    units.push({
+      type: "activity",
+      messages: runMessages,
+      items: runMessages.flatMap(activityItemsForMessage),
+      turnLatencyMs: activityTurnLatencyMs(runMessages, visibleMessages),
+    });
+    runMessages = [];
+    runBucket = undefined;
+  };
+
+  for (const message of activityMessages) {
+    const bucket = isFileEditActivityMessage(message) ? "file" : "other";
+    if (runBucket && bucket !== runBucket) {
+      flushRun();
+    }
+    runBucket = bucket;
+    runMessages.push(message);
+  }
+
+  flushRun();
+}
+
+function isFileEditActivityMessage(message: UIMessage): boolean {
+  return message.kind === "trace" && !!message.fileEdits?.length;
+}
+
+function assistantHasInlineReasoning(message: UIMessage): boolean {
+  return (
+    message.role === "assistant"
+    && message.kind !== "trace"
+    && message.content.trim().length > 0
+    && (!!message.reasoning?.trim() || !!message.reasoningStreaming)
+  );
+}
+
+function reasoningOnlyMessageFromAnswer(message: UIMessage): UIMessage {
+  return {
+    id: `${message.id}-reasoning`,
+    role: "assistant",
+    content: "",
+    createdAt: message.createdAt,
+    reasoning: message.reasoning,
+    reasoningStreaming: message.reasoningStreaming,
+    isStreaming: message.reasoningStreaming,
+    activitySegmentId: message.activitySegmentId,
+    latencyMs: message.latencyMs,
+  };
+}
+
+function stripInlineReasoning(message: UIMessage): UIMessage {
+  const next = { ...message };
+  delete next.reasoning;
+  delete next.reasoningStreaming;
+  return next;
+}
+
+function activityItemsForMessage(message: UIMessage): ActivityItem[] {
+  if (isReasoningOnlyAssistant(message)) {
+    return [{ type: "reasoning", message }];
+  }
+  if (message.kind !== "trace") return [];
+
+  const items: ActivityItem[] = [];
+  if (message.fileEdits?.length) {
+    items.push({ type: "file_edit", message });
+  }
+  for (const event of message.toolEvents ?? []) {
+    const name = String(event.name ?? "").toLowerCase();
+    if (name === "run_cli_app") {
+      items.push({ type: "cli", message });
+    } else if (name === "mcp") {
+      items.push({ type: "mcp", message });
+    } else {
+      items.push({ type: "tool", message });
+    }
+  }
+  if (items.length === 0 && (message.traces?.length || message.content.trim())) {
+    items.push({ type: "tool", message });
+  }
+  if (message.media?.length) {
+    items.push({ type: "media", message });
+  }
+  return items;
+}
+
+function activityTurnLatencyMs(activityMessages: UIMessage[], visibleMessages: UIMessage[]): number | undefined {
+  for (let i = activityMessages.length - 1; i >= 0; i -= 1) {
+    const latency = activityMessages[i].latencyMs;
+    if (isValidLatency(latency)) return latency;
+  }
+  for (let i = visibleMessages.length - 1; i >= 0; i -= 1) {
+    const latency = visibleMessages[i].latencyMs;
+    if (isValidLatency(latency)) return latency;
+  }
+  return undefined;
+}
+
+function isValidLatency(value: unknown): value is number {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+
+export function activityEvidenceFromToolEvent(event: ToolProgressEvent): ActivityEvidence[] {
+  const source = activitySourceFromToolName(toolEventName(event));
+  const evidence: ActivityEvidence[] = [];
+  const extras = [
+    ...unknownList((event as { embeds?: unknown }).embeds),
+    ...unknownList((event as { files?: unknown }).files),
+  ];
+  extras.forEach((value, index) => {
+    const attachment = mediaAttachmentFromUnknown(value);
+    if (!attachment) return;
+    evidence.push({
+      id: `${event.call_id || toolEventName(event) || "tool"}:${index}:${attachment.url || attachment.name || attachment.kind}`,
+      attachment,
+      caption: attachment.name,
+      source,
+    });
+  });
+  return evidence;
+}
+
+export function activityEvidenceFromMessageMedia(message: UIMessage): ActivityEvidence[] {
+  return (message.media ?? []).map((attachment, index) => ({
+    id: `${message.id}:media:${index}:${attachment.url || attachment.name || attachment.kind}`,
+    attachment,
+    caption: attachment.name,
+    source: "media",
+  }));
+}
+
+function unknownList(value: unknown): unknown[] {
+  return Array.isArray(value) ? value : [];
+}
+
+function toolEventName(event: ToolProgressEvent): string {
+  return typeof (event as { function?: { name?: unknown } }).function?.name === "string"
+    ? String((event as { function?: { name?: unknown } }).function?.name)
+    : typeof event.name === "string"
+      ? event.name
+      : "";
+}
+
+function activitySourceFromToolName(name: string): ActivityStepSource {
+  const compact = name.toLowerCase();
+  if (compact.includes("browser") || compact.includes("screenshot")) return "browser";
+  if (compact.includes("web") || compact.includes("search") || compact.includes("fetch") || compact.includes("read")) return "web";
+  if (compact.includes("exec") || compact.includes("shell") || compact.includes("cli")) return "shell";
+  if (compact.startsWith("mcp_") || compact === "mcp") return "mcp";
+  if (compact.includes("file") || compact.includes("patch")) return "file";
+  if (compact.includes("image") || compact.includes("video") || compact.includes("media")) return "media";
+  return "tool";
+}
+
+function mediaAttachmentFromUnknown(value: unknown): UIMediaAttachment | null {
+  if (typeof value === "string") {
+    const text = value.trim();
+    if (!text) return null;
+    return toMediaAttachment({ url: looksLikeUrl(text) ? text : undefined, name: baseName(text) });
+  }
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const record = value as Record<string, unknown>;
+  const url = stringField(record, ["url", "href", "src", "uri", "signed_url", "thumbnail_url"]);
+  const path = stringField(record, ["path", "absolute_path", "file", "filename"]);
+  const name = stringField(record, ["name", "filename", "title", "label"]) ?? baseName(url ?? path ?? "");
+  const kind = mediaKindFromRecord(record, url, name);
+  return toMediaAttachment({ url, name, kind });
+}
+
+function stringField(record: Record<string, unknown>, keys: string[]): string | undefined {
+  for (const key of keys) {
+    const value = record[key];
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return undefined;
+}
+
+function mediaKindFromRecord(record: Record<string, unknown>, url?: string, name?: string): UIMediaAttachment["kind"] | undefined {
+  const raw = stringField(record, ["kind", "type", "mime", "mime_type", "content_type"])?.toLowerCase() ?? "";
+  if (raw.includes("image") || raw.includes("screenshot")) return "image";
+  if (raw.includes("video") || raw.includes("mp4") || raw.includes("quicktime")) return "video";
+  if (raw.includes("file") || raw.includes("document")) return "file";
+  return toMediaAttachment({ url, name }).kind;
+}
+
+function looksLikeUrl(value: string): boolean {
+  return /^(https?:|data:|\/api\/|blob:)/i.test(value);
+}
+
+function baseName(value: string): string | undefined {
+  const clean = value.split(/[?#]/, 1)[0] ?? "";
+  const last = clean.split(/[\\/]/).filter(Boolean).pop();
+  return last || undefined;
+}

webui/src/lib/api.ts

@@ -6,6 +6,7 @@ import type {
   ModelConfigurationCreate,
   ModelConfigurationUpdate,
   NetworkSafetySettingsUpdate,
+  ProviderModelsPayload,
   ProviderSettingsUpdate,
   SettingsPayload,
   SettingsUpdate,
@@ -174,6 +175,19 @@ export async function fetchMcpPresets(
   return request<McpPresetsPayload>(`${base}/api/settings/mcp-presets`, token);
 }
 
+export async function fetchProviderModels(
+  token: string,
+  provider: string,
+  base: string = "",
+): Promise<ProviderModelsPayload> {
+  const query = new URLSearchParams();
+  query.set("provider", provider);
+  return request<ProviderModelsPayload>(
+    `${base}/api/settings/provider-models?${query}`,
+    token,
+  );
+}
+
 export async function runMcpPresetAction(
   token: string,
   action: "enable" | "remove" | "test",

webui/src/lib/media.ts

@@ -8,6 +8,7 @@ const IMAGE_EXTENSIONS = new Set([
   ".webp",
   ".bmp",
   ".ico",
+  ".svg",
   ".tif",
   ".tiff",
 ]);
@@ -34,26 +35,30 @@ function extensionOf(value?: string): string {
   return path.slice(dot);
 }
 
-export function inferMediaKind(media: { url?: string; name?: string }): UIMediaKind {
+function explicitMediaKind(media: { url?: string; name?: string }): UIMediaKind | null {
   const url = media.url ?? "";
   if (url.startsWith("data:image/")) return "image";
   if (url.startsWith("data:video/")) return "video";
 
   const ext = extensionOf(media.name) || extensionOf(url);
+  if (!ext) return null;
   if (IMAGE_EXTENSIONS.has(ext)) return "image";
   if (VIDEO_EXTENSIONS.has(ext)) return "video";
   return "file";
 }
 
+export function inferMediaKind(media: { url?: string; name?: string }): UIMediaKind {
+  return explicitMediaKind(media) ?? "file";
+}
+
 export function toMediaAttachment(media: {
   url?: string;
   name?: string;
   kind?: UIMediaKind;
 }): UIMediaAttachment {
   return {
-    kind: media.kind ?? inferMediaKind(media),
+    kind: explicitMediaKind(media) ?? media.kind ?? "file",
     url: media.url,
     name: media.name,
   };
 }
-

webui/src/lib/provider-brand.ts

@@ -146,7 +146,9 @@ const PROVIDER_BRANDS: Record<string, ProviderBrand> = {
   searxng: brand("searxng.org", "#3050FF", "SX"),
   siliconflow: brand("siliconflow.cn", "#111827", "SF"),
   skywork: brand("skywork.ai", "#5B5BF6", "SW"),
-  stepfun: brand("stepfun.com", "#2F6BFF", "SF"),
+  stepfun: brand("stepfun.com", "#2F6BFF", "SF", [
+    "https://www.stepfun.com/step_favicon.svg",
+  ]),
   tavily: brand("tavily.com", "#111827", "T"),
   volcengine: brand("volcengine.com", "#1664FF", "VE"),
   vllm: brand("vllm.ai", "#2563EB", "VL"),

webui/src/lib/types.ts

@@ -221,6 +221,29 @@ export interface RuntimeCapabilities {
   can_export_diagnostics: boolean;
 }
 
+export interface ProviderModelInfo {
+  id: string;
+  label?: string | null;
+  owned_by?: string | null;
+  context_window?: number | null;
+}
+
+export interface ProviderModelsPayload {
+  provider: string;
+  label: string;
+  status:
+    | "available"
+    | "unsupported"
+    | "not_configured"
+    | "missing_api_base"
+    | "error";
+  catalog_kind: "official" | "catalog" | "local" | "custom" | "unsupported";
+  models: ProviderModelInfo[];
+  model_count: number;
+  message?: string | null;
+  fetched_at?: number;
+}
+
 export interface SettingsPayload {
   surface?: RuntimeSurface;
   runtime_surface?: RuntimeSurface;