feat(transcription): add shared voice input support (#4232)

* feat(webui): add voice transcription input

* feat(webui): render ANSI output in code blocks

* refactor(webui): isolate voice recorder logic

* refactor(transcription): keep websocket ingress thin

* refactor(transcription): resolve channel audio settings on demand

* style(webui): neutralize voice waveform color

* feat(webui): add voice input tooltip

* feat(webui): add voice input keyboard shortcut

* fix(webui): distinguish voice shortcut platforms

* fix(webui): place voice button after model selector

* refactor(webui): share voice hold recording helpers

* fix(desktop): allow microphone voice input

* fix(webui): stabilize token usage month labels

* feat(webui): show voice input on settings overview

* fix(webui): label voice capability as recognition

* fix(webui): align capability overview status

* refactor(webui): isolate transcription socket handling

* fix(webui): soften silent voice waveform

* refactor(audio): clarify transcription service location

* docs(transcription): clarify audio and provider boundaries

* fix(exec): reduce session output polling flake

webui/src/tests/app-layout.test.tsx

@@ -1172,13 +1172,13 @@ describe("App layout", () => {
 
   it("restores the settings section from the URL hash after a page reload", async () => {
     mockFetchRoutes({ "/api/settings": baseSettingsPayload() });
-    window.history.replaceState(null, "", "/#/settings?section=models");
+    window.history.replaceState(null, "", "/#/settings?section=voice");
 
     render(<App />);
 
     await waitFor(() => expect(connectSpy).toHaveBeenCalled());
-    expect(await screen.findByRole("heading", { name: "Models" })).toBeInTheDocument();
-    expect(window.location.hash).toBe("#/settings?section=models");
+    expect(await screen.findByRole("heading", { name: "Voice input" })).toBeInTheDocument();
+    expect(window.location.hash).toBe("#/settings?section=voice");
   });
 
   it("updates the URL hash when switching settings sections", async () => {
@@ -1197,6 +1197,11 @@ describe("App layout", () => {
 
     expect(await screen.findByRole("heading", { name: "Models" })).toBeInTheDocument();
     expect(window.location.hash).toBe("#/settings?section=models");
+
+    fireEvent.click(within(settingsNav).getByRole("button", { name: "Voice" }));
+
+    expect(await screen.findByRole("heading", { name: "Voice input" })).toBeInTheDocument();
+    expect(window.location.hash).toBe("#/settings?section=voice");
   });
 
   it("opens Apps from the main sidebar without replacing the sidebar", async () => {

webui/src/tests/code-block.test.tsx

@@ -1,4 +1,5 @@
 import { act, render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
 import { describe, expect, it, vi } from "vitest";
 
 import { CodeBlock } from "@/components/CodeBlock";
@@ -87,6 +88,64 @@ describe("CodeBlock", () => {
     expect(screen.getByText("const value = 1;")).toBeInTheDocument();
   });
 
+  it("renders ANSI output without mounting the syntax highlighter", () => {
+    render(
+      <ThemeProvider theme="dark">
+        <CodeBlock
+          language="ansi"
+          code={"\x1b[32mPASS\x1b[0m <script>alert(1)</script>"}
+        />
+      </ThemeProvider>,
+    );
+
+    expect(screen.queryByTestId("highlighted-code")).not.toBeInTheDocument();
+    expect(screen.getByTestId("ansi-code")).toBeInTheDocument();
+    expect(screen.getByTestId("ansi-code").closest(".not-prose")).toBeTruthy();
+    expect(screen.getByText("ansi")).toBeInTheDocument();
+    expect(screen.getByText("PASS")).toHaveStyle({ color: "#0dbc79" });
+    expect(screen.getByText("<script>alert(1)</script>")).toBeInTheDocument();
+    expect(document.querySelector("script")).toBeNull();
+  });
+
+  it("detects ANSI sequences in regular code blocks", () => {
+    render(
+      <ThemeProvider theme="light">
+        <CodeBlock
+          language="text"
+          code={"\x1b[38;2;35;209;139mtruecolor\x1b[0m"}
+        />
+      </ThemeProvider>,
+    );
+
+    expect(screen.queryByTestId("highlighted-code")).not.toBeInTheDocument();
+    expect(screen.getByText("truecolor")).toHaveStyle({
+      color: "rgb(35, 209, 139)",
+    });
+  });
+
+  it("copies ANSI output as clean text", async () => {
+    const user = userEvent.setup();
+    const writeText = vi.fn().mockResolvedValue(undefined);
+    Object.defineProperty(navigator, "clipboard", {
+      configurable: true,
+      value: { writeText },
+    });
+
+    try {
+      render(
+        <ThemeProvider theme="dark">
+          <CodeBlock language="ansi" code={"\x1b[32mPASS\x1b[0m"} />
+        </ThemeProvider>,
+      );
+
+      await user.click(screen.getByRole("button", { name: /copy/i }));
+
+      expect(writeText).toHaveBeenCalledWith("PASS");
+    } finally {
+      Reflect.deleteProperty(navigator, "clipboard");
+    }
+  });
+
   it("reads theme from context without creating per-block observers", async () => {
     const originalMutationObserver = globalThis.MutationObserver;
     const observer = vi.fn();

webui/src/tests/nanobot-client.test.ts

@@ -412,6 +412,61 @@ describe("NanobotClient", () => {
     );
   });
 
+  it("sends transcription requests and resolves transcription results outside chat dispatch", async () => {
+    const client = new NanobotClient({
+      url: "ws://test",
+      reconnect: false,
+      socketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
+    });
+    const handler = vi.fn();
+    client.onChat("chat-a", handler);
+    client.connect();
+    lastSocket().fakeOpen();
+
+    const promise = client.transcribeAudio("data:audio/webm;base64,AAAA", {
+      durationMs: 1234,
+      timeoutMs: 1_000,
+    });
+    const frame = JSON.parse(lastSocket().sent.at(-1) as string);
+    expect(frame).toMatchObject({
+      type: "transcribe_audio",
+      data_url: "data:audio/webm;base64,AAAA",
+      duration_ms: 1234,
+    });
+    expect(typeof frame.request_id).toBe("string");
+
+    lastSocket().fakeMessage({
+      event: "transcription_result",
+      request_id: frame.request_id,
+      text: "hello from voice",
+    });
+    await expect(promise).resolves.toBe("hello from voice");
+    expect(handler).not.toHaveBeenCalled();
+  });
+
+  it("rejects pending transcription requests on server errors and socket close", async () => {
+    const client = new NanobotClient({
+      url: "ws://test",
+      reconnect: false,
+      socketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
+    });
+    client.connect();
+    lastSocket().fakeOpen();
+
+    const errored = client.transcribeAudio("data:audio/webm;base64,AAAA", { timeoutMs: 1_000 });
+    const errorFrame = JSON.parse(lastSocket().sent.at(-1) as string);
+    lastSocket().fakeMessage({
+      event: "transcription_error",
+      request_id: errorFrame.request_id,
+      detail: "not_configured",
+    });
+    await expect(errored).rejects.toThrow("not_configured");
+
+    const dropped = client.transcribeAudio("data:audio/webm;base64,BBBB", { timeoutMs: 1_000 });
+    lastSocket().close();
+    await expect(dropped).rejects.toThrow("socket closed");
+  });
+
   it("queues sends while connecting and flushes on open", () => {
     const client = new NanobotClient({
       url: "ws://test",

webui/src/tests/thread-composer.test.tsx

@@ -1,4 +1,4 @@
-import { fireEvent, render, screen, waitFor, within } from "@testing-library/react";
+import { act, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
 import { afterEach, describe, expect, it, vi } from "vitest";
 
 import { ThreadComposer } from "@/components/thread/ThreadComposer";
@@ -121,6 +121,7 @@ const MCP_PRESETS: McpPresetInfo[] = [
   },
 ];
 const ORIGINAL_INNER_HEIGHT = window.innerHeight;
+const ORIGINAL_MEDIA_DEVICES = navigator.mediaDevices;
 
 function mockBlobUrls() {
   Object.defineProperty(URL, "createObjectURL", {
@@ -135,7 +136,16 @@ function mockBlobUrls() {
 
 afterEach(() => {
   vi.restoreAllMocks();
+  vi.unstubAllGlobals();
   Reflect.deleteProperty(window, "nanobotHost");
+  if (ORIGINAL_MEDIA_DEVICES) {
+    Object.defineProperty(navigator, "mediaDevices", {
+      configurable: true,
+      value: ORIGINAL_MEDIA_DEVICES,
+    });
+  } else {
+    Reflect.deleteProperty(navigator, "mediaDevices");
+  }
   window.localStorage.clear();
   Object.defineProperty(window, "innerHeight", {
     value: ORIGINAL_INNER_HEIGHT,
@@ -161,6 +171,75 @@ function rect(init: Partial<DOMRect>): DOMRect {
   };
 }
 
+function mockVoiceRecorder(blob = new Blob(["voice"], { type: "audio/webm" })) {
+  const stopTrack = vi.fn();
+  const getUserMedia = vi.fn(async () => ({
+    getTracks: () => [{ stop: stopTrack }],
+  }));
+  Object.defineProperty(navigator, "mediaDevices", {
+    configurable: true,
+    value: { getUserMedia },
+  });
+
+  class FakeMediaRecorder {
+    static isTypeSupported = vi.fn((type: string) => type === "audio/webm");
+
+    state: RecordingState = "inactive";
+    mimeType = blob.type;
+    ondataavailable: ((event: BlobEvent) => void) | null = null;
+    onstop: (() => void) | null = null;
+
+    start() {
+      this.state = "recording";
+    }
+
+    stop() {
+      this.state = "inactive";
+      this.ondataavailable?.({ data: blob } as BlobEvent);
+      this.onstop?.();
+    }
+  }
+
+  vi.stubGlobal("MediaRecorder", FakeMediaRecorder);
+  return { getUserMedia, stopTrack };
+}
+
+function mockVoiceAudioInput(sample = 128, state: AudioContextState = "running") {
+  class FakeAudioContext {
+    state = state;
+
+    createMediaStreamSource() {
+      return { connect: vi.fn(), disconnect: vi.fn() };
+    }
+
+    createAnalyser() {
+      return {
+        fftSize: 256,
+        smoothingTimeConstant: 0,
+        disconnect: vi.fn(),
+        getByteTimeDomainData: (data: Uint8Array) => data.fill(sample),
+      };
+    }
+
+    close = vi.fn(async () => undefined);
+    resume = vi.fn(async () => undefined);
+  }
+
+  vi.stubGlobal("AudioContext", FakeAudioContext);
+  vi.spyOn(window, "requestAnimationFrame").mockImplementation((callback) =>
+    window.setTimeout(() => callback(performance.now()), 16) as unknown as number
+  );
+  vi.spyOn(window, "cancelAnimationFrame").mockImplementation((id) =>
+    window.clearTimeout(id as unknown as number)
+  );
+}
+
+async function waitForVoiceCapture(): Promise<void> {
+  await act(async () => {
+    await new Promise((resolve) => setTimeout(resolve, 700));
+  });
+}
+
 describe("ThreadComposer", () => {
   it("renders a readonly hero model composer when provided", () => {
     render(
@@ -209,6 +288,245 @@ describe("ThreadComposer", () => {
     expect(screen.queryByText(/Enter to send/)).not.toBeInTheDocument();
   });
 
+  it("transcribes voice input into the composer without sending", async () => {
+    mockVoiceRecorder();
+    const onSend = vi.fn();
+    const onTranscribeAudio = vi.fn(async () => "hello voice");
+    render(
+      <ThreadComposer
+        onSend={onSend}
+        onTranscribeAudio={onTranscribeAudio}
+        placeholder="Type your message..."
+      />,
+    );
+
+    fireEvent.click(screen.getByRole("button", { name: "Voice input" }));
+    expect(await screen.findByLabelText("Recording 0:00")).toBeInTheDocument();
+    await waitForVoiceCapture();
+    fireEvent.click(await screen.findByRole("button", { name: "Stop recording" }));
+
+    await waitFor(() => expect(onTranscribeAudio).toHaveBeenCalledWith(
+      expect.stringMatching(/^data:audio\/webm;base64,/),
+      expect.objectContaining({ durationMs: expect.any(Number) }),
+    ));
+    await waitFor(() => expect(screen.getByLabelText("Message input")).toHaveValue("hello voice"));
+    expect(onSend).not.toHaveBeenCalled();
+  });
+
+  it("does not start duplicate voice recordings while microphone access is pending", async () => {
+    const { getUserMedia, stopTrack } = mockVoiceRecorder();
+    let resolveStream: ((stream: MediaStream) => void) | undefined;
+    getUserMedia.mockImplementation(() => new Promise((resolve) => {
+      resolveStream = resolve as (stream: MediaStream) => void;
+    }));
+    const onTranscribeAudio = vi.fn(async () => "one recording");
+    render(
+      <ThreadComposer
+        onSend={vi.fn()}
+        onTranscribeAudio={onTranscribeAudio}
+        placeholder="Type your message..."
+      />,
+    );
+
+    const voiceButton = screen.getByRole("button", { name: "Voice input" });
+    fireEvent.click(voiceButton);
+    fireEvent.click(voiceButton);
+
+    expect(getUserMedia).toHaveBeenCalledTimes(1);
+
+    await act(async () => {
+      resolveStream?.({ getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream);
+    });
+    expect(await screen.findByLabelText("Recording 0:00")).toBeInTheDocument();
+    await waitForVoiceCapture();
+    fireEvent.click(await screen.findByRole("button", { name: "Stop recording" }));
+
+    await waitFor(() => expect(onTranscribeAudio).toHaveBeenCalledTimes(1));
+    await waitFor(() => expect(screen.getByLabelText("Message input")).toHaveValue("one recording"));
+  });
+
+  it("supports press-and-hold voice recording", async () => {
+    mockVoiceRecorder();
+    const onSend = vi.fn();
+    const onTranscribeAudio = vi.fn(async () => "held voice");
+    render(
+      <ThreadComposer
+        onSend={onSend}
+        onTranscribeAudio={onTranscribeAudio}
+        placeholder="Type your message..."
+      />,
+    );
+
+    const voiceButton = screen.getByRole("button", { name: "Voice input" });
+    fireEvent.pointerDown(voiceButton, { button: 0, pointerId: 1, pointerType: "touch" });
+    await act(async () => {
+      await new Promise((resolve) => setTimeout(resolve, 180));
+    });
+    expect(await screen.findByLabelText("Recording 0:00")).toBeInTheDocument();
+    await waitForVoiceCapture();
+    fireEvent.pointerUp(screen.getByRole("button", { name: "Stop recording" }), {
+      pointerId: 1,
+      pointerType: "touch",
+    });
+
+    await waitFor(() => expect(onTranscribeAudio).toHaveBeenCalled());
+    await waitFor(() => expect(screen.getByLabelText("Message input")).toHaveValue("held voice"));
+    expect(onSend).not.toHaveBeenCalled();
+  });
+
+  it("supports keyboard hold voice recording", async () => {
+    mockVoiceRecorder();
+    const onSend = vi.fn();
+    const onTranscribeAudio = vi.fn(async () => "shortcut voice");
+    render(
+      <ThreadComposer
+        onSend={onSend}
+        onTranscribeAudio={onTranscribeAudio}
+        placeholder="Type your message..."
+      />,
+    );
+
+    const voiceButton = screen.getByRole("button", { name: "Voice input" });
+    expect(voiceButton).toHaveAttribute("title", "Click to dictate or hold");
+    expect(voiceButton).toHaveAttribute("aria-keyshortcuts", "Control+Shift+D");
+    fireEvent.keyDown(window, { code: "KeyD", ctrlKey: true, key: "D", shiftKey: true });
+    expect(await screen.findByLabelText("Recording 0:00")).toBeInTheDocument();
+    await waitForVoiceCapture();
+    fireEvent.keyUp(window, { code: "KeyD", ctrlKey: true, key: "D", shiftKey: true });
+
+    await waitFor(() => expect(onTranscribeAudio).toHaveBeenCalled());
+    await waitFor(() => expect(screen.getByLabelText("Message input")).toHaveValue("shortcut voice"));
+    expect(onSend).not.toHaveBeenCalled();
+  });
+
+  it("ignores the delayed click emitted after a long-press voice recording", async () => {
+    const { getUserMedia } = mockVoiceRecorder();
+    const onTranscribeAudio = vi.fn(async () => "held once");
+    render(
+      <ThreadComposer
+        onSend={vi.fn()}
+        onTranscribeAudio={onTranscribeAudio}
+        placeholder="Type your message..."
+      />,
+    );
+
+    const voiceButton = screen.getByRole("button", { name: "Voice input" });
+    fireEvent.pointerDown(voiceButton, { button: 0, pointerId: 1, pointerType: "touch" });
+    await act(async () => {
+      await new Promise((resolve) => setTimeout(resolve, 180));
+    });
+    expect(await screen.findByLabelText("Recording 0:00")).toBeInTheDocument();
+    await waitForVoiceCapture();
+    fireEvent.pointerUp(screen.getByRole("button", { name: "Stop recording" }), {
+      pointerId: 1,
+      pointerType: "touch",
+    });
+    await waitFor(() => expect(screen.getByLabelText("Message input")).toHaveValue("held once"));
+
+    await act(async () => {
+      await new Promise((resolve) => setTimeout(resolve, 20));
+    });
+    fireEvent.click(screen.getByRole("button", { name: "Voice input" }));
+
+    expect(getUserMedia).toHaveBeenCalledTimes(1);
+    expect(onTranscribeAudio).toHaveBeenCalledTimes(1);
+  });
+
+  it("keeps existing text when voice transcription fails", async () => {
+    mockVoiceRecorder();
+    const onSend = vi.fn();
+    const onTranscribeAudio = vi.fn(async () => {
+      throw new Error("not_configured");
+    });
+    render(
+      <ThreadComposer
+        onSend={onSend}
+        onTranscribeAudio={onTranscribeAudio}
+        placeholder="Type your message..."
+      />,
+    );
+
+    const input = screen.getByLabelText("Message input");
+    fireEvent.change(input, { target: { value: "draft" } });
+    fireEvent.click(screen.getByRole("button", { name: "Voice input" }));
+    await waitForVoiceCapture();
+    fireEvent.click(await screen.findByRole("button", { name: "Stop recording" }));
+
+    await waitFor(() => {
+      expect(screen.getByText("Configure a transcription provider first.")).toBeInTheDocument();
+    });
+    expect(input).toHaveValue("draft");
+    expect(onSend).not.toHaveBeenCalled();
+  });
+
+  it("does not transcribe recordings that are too short", async () => {
+    mockVoiceRecorder();
+    const onTranscribeAudio = vi.fn(async () => "should not appear");
+    render(
+      <ThreadComposer
+        onSend={vi.fn()}
+        onTranscribeAudio={onTranscribeAudio}
+        placeholder="Type your message..."
+      />,
+    );
+
+    fireEvent.click(screen.getByRole("button", { name: "Voice input" }));
+    fireEvent.click(await screen.findByRole("button", { name: "Stop recording" }));
+
+    await waitFor(() => {
+      expect(screen.getByText("Hold a little longer to record voice.")).toBeInTheDocument();
+    });
+    expect(onTranscribeAudio).not.toHaveBeenCalled();
+  });
+
+  it("warns during recording when microphone input is silent", async () => {
+    mockVoiceRecorder();
+    mockVoiceAudioInput();
+    const onTranscribeAudio = vi.fn(async () => "should not appear");
+    render(
+      <ThreadComposer
+        onSend={vi.fn()}
+        onTranscribeAudio={onTranscribeAudio}
+        placeholder="Type your message..."
+      />,
+    );
+
+    fireEvent.click(screen.getByRole("button", { name: "Voice input" }));
+    expect(await screen.findByLabelText("Recording 0:00")).toBeInTheDocument();
+    await act(async () => {
+      await new Promise((resolve) => setTimeout(resolve, 1_150));
+    });
+
+    expect(screen.getByText("No microphone input detected.")).toBeInTheDocument();
+    fireEvent.click(await screen.findByRole("button", { name: "Stop recording" }));
+    expect(onTranscribeAudio).not.toHaveBeenCalled();
+  });
+
+  it("does not treat unavailable microphone levels as silence", async () => {
+    mockVoiceRecorder();
+    mockVoiceAudioInput(128, "suspended");
+    const onTranscribeAudio = vi.fn(async () => "voice text");
+    render(
+      <ThreadComposer
+        onSend={vi.fn()}
+        onTranscribeAudio={onTranscribeAudio}
+        placeholder="Type your message..."
+      />,
+    );
+
+    fireEvent.click(screen.getByRole("button", { name: "Voice input" }));
+    expect(await screen.findByLabelText("Recording 0:00")).toBeInTheDocument();
+    await act(async () => {
+      await new Promise((resolve) => setTimeout(resolve, 1_150));
+    });
+
+    expect(screen.queryByText("No microphone input detected.")).not.toBeInTheDocument();
+    fireEvent.click(await screen.findByRole("button", { name: "Stop recording" }));
+
+    await waitFor(() => expect(onTranscribeAudio).toHaveBeenCalledTimes(1));
+    expect(screen.getByDisplayValue("voice text")).toBeInTheDocument();
+  });
+
   it("renders and changes workspace access mode", async () => {
     const onWorkspaceScopeChange = vi.fn();
     render(