feat(transcription): add shared voice input support (#4232)

* feat(webui): add voice transcription input

* feat(webui): render ANSI output in code blocks

* refactor(webui): isolate voice recorder logic

* refactor(transcription): keep websocket ingress thin

* refactor(transcription): resolve channel audio settings on demand

* style(webui): neutralize voice waveform color

* feat(webui): add voice input tooltip

* feat(webui): add voice input keyboard shortcut

* fix(webui): distinguish voice shortcut platforms

* fix(webui): place voice button after model selector

* refactor(webui): share voice hold recording helpers

* fix(desktop): allow microphone voice input

* fix(webui): stabilize token usage month labels

* feat(webui): show voice input on settings overview

* fix(webui): label voice capability as recognition

* fix(webui): align capability overview status

* refactor(webui): isolate transcription socket handling

* fix(webui): soften silent voice waveform

* refactor(audio): clarify transcription service location

* docs(transcription): clarify audio and provider boundaries

* fix(exec): reduce session output polling flake

webui/src/lib/ansi.ts

@@ -0,0 +1,210 @@
+export type AnsiSegment = {
+  text: string;
+  style?: AnsiStyle;
+};
+
+export type AnsiStyle = {
+  backgroundColor?: string;
+  color?: string;
+  fontStyle?: "italic";
+  fontWeight?: number;
+  opacity?: number;
+  textDecorationLine?: "underline";
+};
+
+type AnsiState = {
+  backgroundColor?: string;
+  bold: boolean;
+  color?: string;
+  dim: boolean;
+  inverse: boolean;
+  italic: boolean;
+  underline: boolean;
+};
+
+const ESC = String.fromCharCode(27);
+const ANSI_PATTERN = new RegExp(`${ESC}\\[[0-?]*[ -/]*[@-~]`, "g");
+
+const ANSI_COLORS = [
+  "#000000",
+  "#cd3131",
+  "#0dbc79",
+  "#e5e510",
+  "#2472c8",
+  "#bc3fbc",
+  "#11a8cd",
+  "#e5e5e5",
+];
+
+const ANSI_BRIGHT_COLORS = [
+  "#666666",
+  "#f14c4c",
+  "#23d18b",
+  "#f5f543",
+  "#3b8eea",
+  "#d670d6",
+  "#29b8db",
+  "#ffffff",
+];
+
+const RGB_STEPS = [0, 95, 135, 175, 215, 255];
+
+export function hasAnsi(value: string): boolean {
+  ANSI_PATTERN.lastIndex = 0;
+  return ANSI_PATTERN.test(value);
+}
+
+export function stripAnsi(value: string): string {
+  ANSI_PATTERN.lastIndex = 0;
+  return value.replace(ANSI_PATTERN, "");
+}
+
+function initialState(): AnsiState {
+  return {
+    bold: false,
+    dim: false,
+    inverse: false,
+    italic: false,
+    underline: false,
+  };
+}
+
+function colorFrom256(value: number): string | undefined {
+  if (value < 0 || value > 255) return undefined;
+  if (value < 8) return ANSI_COLORS[value];
+  if (value < 16) return ANSI_BRIGHT_COLORS[value - 8];
+  if (value < 232) {
+    const offset = value - 16;
+    const red = RGB_STEPS[Math.floor(offset / 36)];
+    const green = RGB_STEPS[Math.floor((offset % 36) / 6)];
+    const blue = RGB_STEPS[offset % 6];
+    return `rgb(${red}, ${green}, ${blue})`;
+  }
+  const gray = 8 + ((value - 232) * 10);
+  return `rgb(${gray}, ${gray}, ${gray})`;
+}
+
+function colorFromRgb(red: number, green: number, blue: number): string | undefined {
+  if ([red, green, blue].some((value) => !Number.isFinite(value) || value < 0 || value > 255)) {
+    return undefined;
+  }
+  return `rgb(${red}, ${green}, ${blue})`;
+}
+
+function normalizedSgrParams(sequence: string): number[] | null {
+  if (!sequence.endsWith("m")) return null;
+  const body = sequence.slice(2, -1).trim();
+  if (!body) return [0];
+  return body.split(/[;:]/).map((part) => {
+    const value = Number.parseInt(part || "0", 10);
+    return Number.isFinite(value) ? value : 0;
+  });
+}
+
+function applyExtendedColor(
+  state: AnsiState,
+  params: number[],
+  index: number,
+  key: "color" | "backgroundColor",
+): number {
+  const mode = params[index + 1];
+  if (mode === 5) {
+    const color = colorFrom256(params[index + 2]);
+    if (color) state[key] = color;
+    return index + 2;
+  }
+  if (mode === 2) {
+    const color = colorFromRgb(params[index + 2], params[index + 3], params[index + 4]);
+    if (color) state[key] = color;
+    return index + 4;
+  }
+  return index;
+}
+
+function applySgrParams(state: AnsiState, params: number[]): void {
+  for (let index = 0; index < params.length; index += 1) {
+    const code = params[index];
+    if (code === 0) {
+      Object.assign(state, initialState());
+    } else if (code === 1) {
+      state.bold = true;
+      state.dim = false;
+    } else if (code === 2) {
+      state.dim = true;
+      state.bold = false;
+    } else if (code === 3) {
+      state.italic = true;
+    } else if (code === 4) {
+      state.underline = true;
+    } else if (code === 7) {
+      state.inverse = true;
+    } else if (code === 22) {
+      state.bold = false;
+      state.dim = false;
+    } else if (code === 23) {
+      state.italic = false;
+    } else if (code === 24) {
+      state.underline = false;
+    } else if (code === 27) {
+      state.inverse = false;
+    } else if (code === 39) {
+      delete state.color;
+    } else if (code === 49) {
+      delete state.backgroundColor;
+    } else if (code >= 30 && code <= 37) {
+      state.color = ANSI_COLORS[code - 30];
+    } else if (code >= 40 && code <= 47) {
+      state.backgroundColor = ANSI_COLORS[code - 40];
+    } else if (code >= 90 && code <= 97) {
+      state.color = ANSI_BRIGHT_COLORS[code - 90];
+    } else if (code >= 100 && code <= 107) {
+      state.backgroundColor = ANSI_BRIGHT_COLORS[code - 100];
+    } else if (code === 38) {
+      index = applyExtendedColor(state, params, index, "color");
+    } else if (code === 48) {
+      index = applyExtendedColor(state, params, index, "backgroundColor");
+    }
+  }
+}
+
+function styleFromState(state: AnsiState): AnsiStyle | undefined {
+  const foreground = state.inverse ? state.backgroundColor : state.color;
+  const background = state.inverse ? state.color : state.backgroundColor;
+  const style: AnsiStyle = {};
+  if (foreground) style.color = foreground;
+  if (background) style.backgroundColor = background;
+  if (state.bold) style.fontWeight = 700;
+  if (state.dim) style.opacity = 0.72;
+  if (state.italic) style.fontStyle = "italic";
+  if (state.underline) style.textDecorationLine = "underline";
+  return Object.keys(style).length ? style : undefined;
+}
+
+export function parseAnsiSegments(value: string): AnsiSegment[] {
+  const segments: AnsiSegment[] = [];
+  const state = initialState();
+  let cursor = 0;
+  ANSI_PATTERN.lastIndex = 0;
+
+  for (const match of value.matchAll(ANSI_PATTERN)) {
+    const index = match.index ?? 0;
+    if (index > cursor) {
+      segments.push({
+        text: value.slice(cursor, index),
+        style: styleFromState(state),
+      });
+    }
+    const params = normalizedSgrParams(match[0]);
+    if (params) applySgrParams(state, params);
+    cursor = index + match[0].length;
+  }
+
+  if (cursor < value.length) {
+    segments.push({
+      text: value.slice(cursor),
+      style: styleFromState(state),
+    });
+  }
+
+  return segments.filter((segment) => segment.text.length > 0);
+}

webui/src/lib/api.ts

@@ -16,6 +16,7 @@ import type {
   SkillDetail,
   SkillsPayload,
   SlashCommand,
+  TranscriptionSettingsUpdate,
   WebSearchSettingsUpdate,
   WorkspacesPayload,
   WebuiThreadPersistedPayload,
@@ -547,3 +548,21 @@ export async function updateImageGenerationSettings(
     token,
   );
 }
+
+export async function updateTranscriptionSettings(
+  token: string,
+  update: TranscriptionSettingsUpdate,
+  base: string = "",
+): Promise<SettingsPayload> {
+  const query = new URLSearchParams();
+  query.set("enabled", String(update.enabled));
+  query.set("provider", update.provider);
+  query.set("model", update.model);
+  query.set("language", update.language);
+  query.set("max_duration_sec", String(update.maxDurationSec));
+  query.set("max_upload_mb", String(update.maxUploadMb));
+  return request<SettingsPayload>(
+    `${base}/api/settings/transcription/update?${query}`,
+    token,
+  );
+}

webui/src/lib/nanobot-client.ts

@@ -95,6 +95,12 @@ interface PendingNewChat {
   timer: ReturnType<typeof setTimeout>;
 }
 
+interface PendingTranscription {
+  resolve: (text: string) => void;
+  reject: (err: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+}
+
 export interface NanobotClientOptions {
   url: string;
   reconnect?: boolean;
@@ -132,6 +138,7 @@ export class NanobotClient {
   /** Latest ``goal_state`` snapshot per ``chat_id`` (multi-session isolation). */
   private goalStateByChatId = new Map<string, GoalStateWsPayload>();
   private pendingNewChat: PendingNewChat | null = null;
+  private pendingTranscriptions = new Map<string, PendingTranscription>();
   // Frames queued while the socket is not yet OPEN
   private sendQueue: Outbound[] = [];
   private reconnectAttempts = 0;
@@ -320,6 +327,27 @@ export class NanobotClient {
     });
   }
 
+  transcribeAudio(
+    dataUrl: string,
+    options?: { durationMs?: number; timeoutMs?: number },
+  ): Promise<string> {
+    const requestId = crypto.randomUUID();
+    const timeoutMs = options?.timeoutMs ?? 120_000;
+    return new Promise<string>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pendingTranscriptions.delete(requestId);
+        reject(new Error("transcription timed out"));
+      }, timeoutMs);
+      this.pendingTranscriptions.set(requestId, { resolve, reject, timer });
+      this.queueSend({
+        type: "transcribe_audio",
+        request_id: requestId,
+        data_url: dataUrl,
+        ...(options?.durationMs !== undefined ? { duration_ms: options.durationMs } : {}),
+      });
+    });
+  }
+
   attach(chatId: string): void {
     this.knownChats.add(chatId);
     if (this.socket?.readyState === WS_OPEN) {
@@ -425,6 +453,16 @@ export class NanobotClient {
       return;
     }
 
+    if (parsed.event === "transcription_result") {
+      this.resolveTranscription(parsed.request_id, parsed.text);
+      return;
+    }
+
+    if (parsed.event === "transcription_error") {
+      this.rejectTranscription(parsed.request_id, parsed.detail || "error");
+      return;
+    }
+
     if (parsed.event === "session_updated") {
       this.emitSessionUpdate(parsed.chat_id, parsed.scope, parsed.workspace_scope);
       return;
@@ -500,6 +538,7 @@ export class NanobotClient {
       this.pendingNewChat.reject(new Error("socket closed"));
       this.pendingNewChat = null;
     }
+    this.rejectAllTranscriptions("socket closed");
     // Surface structured reasons *before* reconnect logic so the UI can
     // display the error even while the client transparently reconnects.
     // Browsers populate ``CloseEvent.code`` with the wire-level close code;
@@ -528,6 +567,34 @@ export class NanobotClient {
     }
   }
 
+  private resolveTranscription(requestId: string, text: string): void {
+    const pending = this.pendingTranscriptions.get(requestId);
+    if (!pending) return;
+    clearTimeout(pending.timer);
+    this.pendingTranscriptions.delete(requestId);
+    pending.resolve(text);
+  }
+
+  private rejectTranscription(requestId: string | undefined, detail: string): void {
+    if (!requestId) {
+      this.rejectAllTranscriptions(detail);
+      return;
+    }
+    const pending = this.pendingTranscriptions.get(requestId);
+    if (!pending) return;
+    clearTimeout(pending.timer);
+    this.pendingTranscriptions.delete(requestId);
+    pending.reject(new Error(detail));
+  }
+
+  private rejectAllTranscriptions(detail: string): void {
+    for (const [requestId, pending] of this.pendingTranscriptions) {
+      clearTimeout(pending.timer);
+      pending.reject(new Error(detail));
+      this.pendingTranscriptions.delete(requestId);
+    }
+  }
+
   private scheduleReconnect(): void {
     this.setStatus("reconnecting");
     const attempt = this.reconnectAttempts++;

webui/src/lib/types.ts

@@ -391,6 +391,23 @@ export interface SettingsPayload {
       default_api_base?: string | null;
     }>;
   };
+  transcription?: {
+    enabled: boolean;
+    provider: string;
+    provider_configured: boolean;
+    model: string;
+    language: string | null;
+    max_duration_sec: number;
+    max_upload_mb: number;
+    providers: Array<{
+      name: string;
+      label: string;
+      configured: boolean;
+      api_key_hint?: string | null;
+      api_base?: string | null;
+      default_api_base?: string | null;
+    }>;
+  };
   runtime: {
     config_path: string;
     workspace_path: string;
@@ -680,6 +697,15 @@ export interface ImageGenerationSettingsUpdate {
   maxImagesPerTurn: number;
 }
 
+export interface TranscriptionSettingsUpdate {
+  enabled: boolean;
+  provider: string;
+  model: string;
+  language: string;
+  maxDurationSec: number;
+  maxUploadMb: number;
+}
+
 export interface SlashCommand {
   command: string;
   title: string;
@@ -782,6 +808,13 @@ export type InboundEvent =
       scope?: "metadata" | "thread" | string;
       workspace_scope?: WorkspaceScopePayload;
     }
+  | { event: "transcription_result"; request_id: string; text: string }
+  | {
+      event: "transcription_error";
+      request_id?: string;
+      detail?: string;
+      provider?: string;
+    }
   | { event: "error"; chat_id?: string; detail?: string; reason?: string };
 
 /** Base64-encoded image attached to an outbound ``message`` envelope.
@@ -845,6 +878,7 @@ export type Outbound =
   | { type: "new_chat"; workspace_scope?: WorkspaceScopePayload }
   | { type: "attach"; chat_id: string }
   | { type: "set_workspace_scope"; chat_id: string; workspace_scope: WorkspaceScopePayload }
+  | { type: "transcribe_audio"; request_id: string; data_url: string; duration_ms?: number }
   | {
       type: "message";
       chat_id: string;