From 28c8c89a42bdf2df1f56dea59130451641a11760 Mon Sep 17 00:00:00 2001 From: zpljd258 <11162658+zpljd258@users.noreply.github.com> Date: Wed, 24 Jun 2026 19:13:11 +0800 Subject: [PATCH] fix(webui): convert WebM to WAV for Xiaomi MiMo ASR transcription MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MiMo ASR (mimo-v2.5-asr) only accepts audio/wav, audio/mp3, and audio/mpeg formats. Web browsers record in WebM/Opus by default, causing the API to reject the payload with a transcription error. This change adds a frontend WebM→WAV converter using the Web Audio API (DecodeAudioData + PCM encoding) that activates only when the configured transcription provider is 'xiaomi_mimo'. Other providers are unaffected — they continue to receive the original browser format. Tested and confirmed working on WebUI. --- .../src/components/thread/ThreadComposer.tsx | 3 + webui/src/components/thread/ThreadShell.tsx | 2 + webui/src/hooks/useVoiceRecorder.ts | 92 ++++++++++++++++++- 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/webui/src/components/thread/ThreadComposer.tsx b/webui/src/components/thread/ThreadComposer.tsx index 5f01cfd0..0ff1301d 100644 --- a/webui/src/components/thread/ThreadComposer.tsx +++ b/webui/src/components/thread/ThreadComposer.tsx @@ -172,6 +172,7 @@ interface ThreadComposerProps { workspaceError?: string | null; onWorkspaceScopeChange?: (scope: WorkspaceScopePayload) => void; pendingQueueKey?: string | null; + transcriptionProvider?: string | null; } const COMMAND_ICONS: Record = { @@ -782,6 +783,7 @@ export function ThreadComposer({ workspaceError = null, onWorkspaceScopeChange, pendingQueueKey = null, + transcriptionProvider = null, }: ThreadComposerProps) { const { t } = useTranslation(); const [value, setValue] = useState(""); @@ -1193,6 +1195,7 @@ export function ThreadComposer({ onError: setVoiceError, onTranscript: appendTranscription, onTranscribeAudio, + wantsWav: transcriptionProvider === "xiaomi_mimo", }); useEffect(() => { diff --git a/webui/src/components/thread/ThreadShell.tsx b/webui/src/components/thread/ThreadShell.tsx index 73c1431f..8def4e5e 100644 --- a/webui/src/components/thread/ThreadShell.tsx +++ b/webui/src/components/thread/ThreadShell.tsx @@ -736,6 +736,7 @@ export function ThreadShell({ workspaceError={workspaceError} onWorkspaceScopeChange={onWorkspaceScopeChange} pendingQueueKey={chatId} + transcriptionProvider={settingsSnapshot?.transcription?.provider} /> ) : ( )} diff --git a/webui/src/hooks/useVoiceRecorder.ts b/webui/src/hooks/useVoiceRecorder.ts index 1ef94048..26803f3d 100644 --- a/webui/src/hooks/useVoiceRecorder.ts +++ b/webui/src/hooks/useVoiceRecorder.ts @@ -42,6 +42,8 @@ interface VoiceRecorderOptions { onError: (key: VoiceRecorderErrorKey) => void; onTranscript: (text: string) => void; onTranscribeAudio?: (dataUrl: string, options?: { durationMs?: number }) => Promise; + /** When true, convert recorded audio to WAV before sending (needed for providers that don't support WebM). */ + wantsWav?: boolean; } export function useVoiceRecorder({ @@ -50,6 +52,7 @@ export function useVoiceRecorder({ onError, onTranscript, onTranscribeAudio, + wantsWav = false, }: VoiceRecorderOptions) { const mediaRecorderRef = useRef(null); const chunksRef = useRef([]); @@ -223,7 +226,9 @@ export function useVoiceRecorder({ return; } setState("transcribing"); - void blobToDataUrl(new Blob(chunks, { type: mimeType })) + const blob = new Blob(chunks, { type: mimeType }); + const audioPromise = wantsWav ? convertBlobToWav(blob) : blobToDataUrl(blob); + void audioPromise .then((dataUrl) => onTranscribeAudio(dataUrl, { durationMs })) .then(onTranscript) .catch((error) => onError(transcriptionErrorKey(error))) @@ -260,6 +265,7 @@ export function useVoiceRecorder({ startWaveform, state, stopRecording, + wantsWav, ]); const startRecordingWithDeferredStop = useCallback(() => { @@ -414,6 +420,90 @@ function blobToDataUrl(blob: Blob): Promise { }); } +/** + * Convert any browser-recorded audio blob (typically webm/opus) to WAV + * using the Web Audio API. This avoids sending unsupported formats + * (e.g. webm) to ASR providers that only accept wav/mp3/mpeg. + */ +async function convertBlobToWav(blob: Blob): Promise { + const AudioCtx = audioContextConstructor(); + if (!AudioCtx) return blobToDataUrl(blob); + + const arrayBuffer = await blob.arrayBuffer(); + const ctx = new AudioCtx(); + try { + const audioBuffer = await ctx.decodeAudioData(arrayBuffer); + const wavBlob = audioBufferToWav(audioBuffer); + return blobToDataUrl(wavBlob); + } finally { + void ctx.close(); + } +} + +/** + * Encode an AudioBuffer as a 16-bit PCM WAV Blob. + */ +function audioBufferToWav(buffer: AudioBuffer): Blob { + const numChannels = buffer.numberOfChannels; + const sampleRate = buffer.sampleRate; + const format = 1; // PCM + const bitsPerSample = 16; + + // Interleave channels + const channels: Float32Array[] = []; + for (let ch = 0; ch < numChannels; ch++) { + channels.push(buffer.getChannelData(ch)); + } + const length = channels[0].length; + const interleaved = new Int16Array(length * numChannels); + for (let i = 0; i < length; i++) { + for (let ch = 0; ch < numChannels; ch++) { + const sample = Math.max(-1, Math.min(1, channels[ch][i])); + interleaved[i * numChannels + ch] = sample < 0 + ? sample * 0x8000 + : sample * 0x7FFF; + } + } + + const byteRate = sampleRate * numChannels * (bitsPerSample / 8); + const blockAlign = numChannels * (bitsPerSample / 8); + const dataSize = interleaved.byteLength; + const headerSize = 44; + const totalSize = headerSize + dataSize; + + const buffer2 = new ArrayBuffer(totalSize); + const view = new DataView(buffer2); + + // RIFF header + writeString(view, 0, "RIFF"); + view.setUint32(4, totalSize - 8, true); + writeString(view, 8, "WAVE"); + + // fmt sub-chunk + writeString(view, 12, "fmt "); + view.setUint32(16, 16, true); // sub-chunk size + view.setUint16(20, format, true); + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, byteRate, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, bitsPerSample, true); + + // data sub-chunk + writeString(view, 36, "data"); + view.setUint32(40, dataSize, true); + + new Int16Array(buffer2, headerSize).set(interleaved); + + return new Blob([buffer2], { type: "audio/wav" }); +} + +function writeString(view: DataView, offset: number, str: string): void { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } +} + function transcriptionErrorKey(error: unknown): VoiceRecorderErrorKey { const detail = error instanceof Error ? error.message : ""; if (detail === "not_configured") return "notConfigured";