feat(transcription): add shared voice input support (#4232)

* feat(webui): add voice transcription input

* feat(webui): render ANSI output in code blocks

* refactor(webui): isolate voice recorder logic

* refactor(transcription): keep websocket ingress thin

* refactor(transcription): resolve channel audio settings on demand

* style(webui): neutralize voice waveform color

* feat(webui): add voice input tooltip

* feat(webui): add voice input keyboard shortcut

* fix(webui): distinguish voice shortcut platforms

* fix(webui): place voice button after model selector

* refactor(webui): share voice hold recording helpers

* fix(desktop): allow microphone voice input

* fix(webui): stabilize token usage month labels

* feat(webui): show voice input on settings overview

* fix(webui): label voice capability as recognition

* fix(webui): align capability overview status

* refactor(webui): isolate transcription socket handling

* fix(webui): soften silent voice waveform

* refactor(audio): clarify transcription service location

* docs(transcription): clarify audio and provider boundaries

* fix(exec): reduce session output polling flake
This commit is contained in:
Xubin Ren
2026-06-09 01:08:49 +08:00
committed by GitHub
parent 06d454a225
commit 9c81280300
49 changed files with 3071 additions and 257 deletions
+67
View File
@@ -95,6 +95,12 @@ interface PendingNewChat {
timer: ReturnType<typeof setTimeout>;
}
interface PendingTranscription {
resolve: (text: string) => void;
reject: (err: Error) => void;
timer: ReturnType<typeof setTimeout>;
}
export interface NanobotClientOptions {
url: string;
reconnect?: boolean;
@@ -132,6 +138,7 @@ export class NanobotClient {
/** Latest ``goal_state`` snapshot per ``chat_id`` (multi-session isolation). */
private goalStateByChatId = new Map<string, GoalStateWsPayload>();
private pendingNewChat: PendingNewChat | null = null;
private pendingTranscriptions = new Map<string, PendingTranscription>();
// Frames queued while the socket is not yet OPEN
private sendQueue: Outbound[] = [];
private reconnectAttempts = 0;
@@ -320,6 +327,27 @@ export class NanobotClient {
});
}
transcribeAudio(
dataUrl: string,
options?: { durationMs?: number; timeoutMs?: number },
): Promise<string> {
const requestId = crypto.randomUUID();
const timeoutMs = options?.timeoutMs ?? 120_000;
return new Promise<string>((resolve, reject) => {
const timer = setTimeout(() => {
this.pendingTranscriptions.delete(requestId);
reject(new Error("transcription timed out"));
}, timeoutMs);
this.pendingTranscriptions.set(requestId, { resolve, reject, timer });
this.queueSend({
type: "transcribe_audio",
request_id: requestId,
data_url: dataUrl,
...(options?.durationMs !== undefined ? { duration_ms: options.durationMs } : {}),
});
});
}
attach(chatId: string): void {
this.knownChats.add(chatId);
if (this.socket?.readyState === WS_OPEN) {
@@ -425,6 +453,16 @@ export class NanobotClient {
return;
}
if (parsed.event === "transcription_result") {
this.resolveTranscription(parsed.request_id, parsed.text);
return;
}
if (parsed.event === "transcription_error") {
this.rejectTranscription(parsed.request_id, parsed.detail || "error");
return;
}
if (parsed.event === "session_updated") {
this.emitSessionUpdate(parsed.chat_id, parsed.scope, parsed.workspace_scope);
return;
@@ -500,6 +538,7 @@ export class NanobotClient {
this.pendingNewChat.reject(new Error("socket closed"));
this.pendingNewChat = null;
}
this.rejectAllTranscriptions("socket closed");
// Surface structured reasons *before* reconnect logic so the UI can
// display the error even while the client transparently reconnects.
// Browsers populate ``CloseEvent.code`` with the wire-level close code;
@@ -528,6 +567,34 @@ export class NanobotClient {
}
}
private resolveTranscription(requestId: string, text: string): void {
const pending = this.pendingTranscriptions.get(requestId);
if (!pending) return;
clearTimeout(pending.timer);
this.pendingTranscriptions.delete(requestId);
pending.resolve(text);
}
private rejectTranscription(requestId: string | undefined, detail: string): void {
if (!requestId) {
this.rejectAllTranscriptions(detail);
return;
}
const pending = this.pendingTranscriptions.get(requestId);
if (!pending) return;
clearTimeout(pending.timer);
this.pendingTranscriptions.delete(requestId);
pending.reject(new Error(detail));
}
private rejectAllTranscriptions(detail: string): void {
for (const [requestId, pending] of this.pendingTranscriptions) {
clearTimeout(pending.timer);
pending.reject(new Error(detail));
this.pendingTranscriptions.delete(requestId);
}
}
private scheduleReconnect(): void {
this.setStatus("reconnecting");
const attempt = this.reconnectAttempts++;