feat(transcription): add shared voice input support (#4232)
* feat(webui): add voice transcription input * feat(webui): render ANSI output in code blocks * refactor(webui): isolate voice recorder logic * refactor(transcription): keep websocket ingress thin * refactor(transcription): resolve channel audio settings on demand * style(webui): neutralize voice waveform color * feat(webui): add voice input tooltip * feat(webui): add voice input keyboard shortcut * fix(webui): distinguish voice shortcut platforms * fix(webui): place voice button after model selector * refactor(webui): share voice hold recording helpers * fix(desktop): allow microphone voice input * fix(webui): stabilize token usage month labels * feat(webui): show voice input on settings overview * fix(webui): label voice capability as recognition * fix(webui): align capability overview status * refactor(webui): isolate transcription socket handling * fix(webui): soften silent voice waveform * refactor(audio): clarify transcription service location * docs(transcription): clarify audio and provider boundaries * fix(exec): reduce session output polling flake
This commit is contained in:
@@ -95,6 +95,12 @@ interface PendingNewChat {
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
}
|
||||
|
||||
interface PendingTranscription {
|
||||
resolve: (text: string) => void;
|
||||
reject: (err: Error) => void;
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
}
|
||||
|
||||
export interface NanobotClientOptions {
|
||||
url: string;
|
||||
reconnect?: boolean;
|
||||
@@ -132,6 +138,7 @@ export class NanobotClient {
|
||||
/** Latest ``goal_state`` snapshot per ``chat_id`` (multi-session isolation). */
|
||||
private goalStateByChatId = new Map<string, GoalStateWsPayload>();
|
||||
private pendingNewChat: PendingNewChat | null = null;
|
||||
private pendingTranscriptions = new Map<string, PendingTranscription>();
|
||||
// Frames queued while the socket is not yet OPEN
|
||||
private sendQueue: Outbound[] = [];
|
||||
private reconnectAttempts = 0;
|
||||
@@ -320,6 +327,27 @@ export class NanobotClient {
|
||||
});
|
||||
}
|
||||
|
||||
transcribeAudio(
|
||||
dataUrl: string,
|
||||
options?: { durationMs?: number; timeoutMs?: number },
|
||||
): Promise<string> {
|
||||
const requestId = crypto.randomUUID();
|
||||
const timeoutMs = options?.timeoutMs ?? 120_000;
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
this.pendingTranscriptions.delete(requestId);
|
||||
reject(new Error("transcription timed out"));
|
||||
}, timeoutMs);
|
||||
this.pendingTranscriptions.set(requestId, { resolve, reject, timer });
|
||||
this.queueSend({
|
||||
type: "transcribe_audio",
|
||||
request_id: requestId,
|
||||
data_url: dataUrl,
|
||||
...(options?.durationMs !== undefined ? { duration_ms: options.durationMs } : {}),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
attach(chatId: string): void {
|
||||
this.knownChats.add(chatId);
|
||||
if (this.socket?.readyState === WS_OPEN) {
|
||||
@@ -425,6 +453,16 @@ export class NanobotClient {
|
||||
return;
|
||||
}
|
||||
|
||||
if (parsed.event === "transcription_result") {
|
||||
this.resolveTranscription(parsed.request_id, parsed.text);
|
||||
return;
|
||||
}
|
||||
|
||||
if (parsed.event === "transcription_error") {
|
||||
this.rejectTranscription(parsed.request_id, parsed.detail || "error");
|
||||
return;
|
||||
}
|
||||
|
||||
if (parsed.event === "session_updated") {
|
||||
this.emitSessionUpdate(parsed.chat_id, parsed.scope, parsed.workspace_scope);
|
||||
return;
|
||||
@@ -500,6 +538,7 @@ export class NanobotClient {
|
||||
this.pendingNewChat.reject(new Error("socket closed"));
|
||||
this.pendingNewChat = null;
|
||||
}
|
||||
this.rejectAllTranscriptions("socket closed");
|
||||
// Surface structured reasons *before* reconnect logic so the UI can
|
||||
// display the error even while the client transparently reconnects.
|
||||
// Browsers populate ``CloseEvent.code`` with the wire-level close code;
|
||||
@@ -528,6 +567,34 @@ export class NanobotClient {
|
||||
}
|
||||
}
|
||||
|
||||
private resolveTranscription(requestId: string, text: string): void {
|
||||
const pending = this.pendingTranscriptions.get(requestId);
|
||||
if (!pending) return;
|
||||
clearTimeout(pending.timer);
|
||||
this.pendingTranscriptions.delete(requestId);
|
||||
pending.resolve(text);
|
||||
}
|
||||
|
||||
private rejectTranscription(requestId: string | undefined, detail: string): void {
|
||||
if (!requestId) {
|
||||
this.rejectAllTranscriptions(detail);
|
||||
return;
|
||||
}
|
||||
const pending = this.pendingTranscriptions.get(requestId);
|
||||
if (!pending) return;
|
||||
clearTimeout(pending.timer);
|
||||
this.pendingTranscriptions.delete(requestId);
|
||||
pending.reject(new Error(detail));
|
||||
}
|
||||
|
||||
private rejectAllTranscriptions(detail: string): void {
|
||||
for (const [requestId, pending] of this.pendingTranscriptions) {
|
||||
clearTimeout(pending.timer);
|
||||
pending.reject(new Error(detail));
|
||||
this.pendingTranscriptions.delete(requestId);
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleReconnect(): void {
|
||||
this.setStatus("reconnecting");
|
||||
const attempt = this.reconnectAttempts++;
|
||||
|
||||
Reference in New Issue
Block a user