fix(webui): convert WebM to WAV for Xiaomi MiMo ASR transcription

MiMo ASR (mimo-v2.5-asr) only accepts audio/wav, audio/mp3, and
audio/mpeg formats. Web browsers record in WebM/Opus by default,
causing the API to reject the payload with a transcription error.

This change adds a frontend WebM→WAV converter using the Web Audio API
(DecodeAudioData + PCM encoding) that activates only when the
configured transcription provider is 'xiaomi_mimo'. Other providers
are unaffected — they continue to receive the original browser format.

Tested and confirmed working on WebUI.
This commit is contained in:
zpljd258
2026-06-25 22:52:55 +08:00
committed by Xubin Ren
parent 7899857201
commit 28c8c89a42
3 changed files with 96 additions and 1 deletions
@@ -172,6 +172,7 @@ interface ThreadComposerProps {
workspaceError?: string | null;
onWorkspaceScopeChange?: (scope: WorkspaceScopePayload) => void;
pendingQueueKey?: string | null;
transcriptionProvider?: string | null;
}
const COMMAND_ICONS: Record<string, LucideIcon> = {
@@ -782,6 +783,7 @@ export function ThreadComposer({
workspaceError = null,
onWorkspaceScopeChange,
pendingQueueKey = null,
transcriptionProvider = null,
}: ThreadComposerProps) {
const { t } = useTranslation();
const [value, setValue] = useState("");
@@ -1193,6 +1195,7 @@ export function ThreadComposer({
onError: setVoiceError,
onTranscript: appendTranscription,
onTranscribeAudio,
wantsWav: transcriptionProvider === "xiaomi_mimo",
});
useEffect(() => {
@@ -736,6 +736,7 @@ export function ThreadShell({
workspaceError={workspaceError}
onWorkspaceScopeChange={onWorkspaceScopeChange}
pendingQueueKey={chatId}
transcriptionProvider={settingsSnapshot?.transcription?.provider}
/>
) : (
<ThreadComposer
@@ -765,6 +766,7 @@ export function ThreadShell({
workspaceScopeDisabled={workspaceScopeDisabled}
workspaceError={workspaceError}
onWorkspaceScopeChange={onWorkspaceScopeChange}
transcriptionProvider={settingsSnapshot?.transcription?.provider}
/>
)}
</>
+91 -1
View File
@@ -42,6 +42,8 @@ interface VoiceRecorderOptions {
onError: (key: VoiceRecorderErrorKey) => void;
onTranscript: (text: string) => void;
onTranscribeAudio?: (dataUrl: string, options?: { durationMs?: number }) => Promise<string>;
/** When true, convert recorded audio to WAV before sending (needed for providers that don't support WebM). */
wantsWav?: boolean;
}
export function useVoiceRecorder({
@@ -50,6 +52,7 @@ export function useVoiceRecorder({
onError,
onTranscript,
onTranscribeAudio,
wantsWav = false,
}: VoiceRecorderOptions) {
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
const chunksRef = useRef<BlobPart[]>([]);
@@ -223,7 +226,9 @@ export function useVoiceRecorder({
return;
}
setState("transcribing");
void blobToDataUrl(new Blob(chunks, { type: mimeType }))
const blob = new Blob(chunks, { type: mimeType });
const audioPromise = wantsWav ? convertBlobToWav(blob) : blobToDataUrl(blob);
void audioPromise
.then((dataUrl) => onTranscribeAudio(dataUrl, { durationMs }))
.then(onTranscript)
.catch((error) => onError(transcriptionErrorKey(error)))
@@ -260,6 +265,7 @@ export function useVoiceRecorder({
startWaveform,
state,
stopRecording,
wantsWav,
]);
const startRecordingWithDeferredStop = useCallback(() => {
@@ -414,6 +420,90 @@ function blobToDataUrl(blob: Blob): Promise<string> {
});
}
/**
* Convert any browser-recorded audio blob (typically webm/opus) to WAV
* using the Web Audio API. This avoids sending unsupported formats
* (e.g. webm) to ASR providers that only accept wav/mp3/mpeg.
*/
async function convertBlobToWav(blob: Blob): Promise<string> {
const AudioCtx = audioContextConstructor();
if (!AudioCtx) return blobToDataUrl(blob);
const arrayBuffer = await blob.arrayBuffer();
const ctx = new AudioCtx();
try {
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
const wavBlob = audioBufferToWav(audioBuffer);
return blobToDataUrl(wavBlob);
} finally {
void ctx.close();
}
}
/**
* Encode an AudioBuffer as a 16-bit PCM WAV Blob.
*/
function audioBufferToWav(buffer: AudioBuffer): Blob {
const numChannels = buffer.numberOfChannels;
const sampleRate = buffer.sampleRate;
const format = 1; // PCM
const bitsPerSample = 16;
// Interleave channels
const channels: Float32Array[] = [];
for (let ch = 0; ch < numChannels; ch++) {
channels.push(buffer.getChannelData(ch));
}
const length = channels[0].length;
const interleaved = new Int16Array(length * numChannels);
for (let i = 0; i < length; i++) {
for (let ch = 0; ch < numChannels; ch++) {
const sample = Math.max(-1, Math.min(1, channels[ch][i]));
interleaved[i * numChannels + ch] = sample < 0
? sample * 0x8000
: sample * 0x7FFF;
}
}
const byteRate = sampleRate * numChannels * (bitsPerSample / 8);
const blockAlign = numChannels * (bitsPerSample / 8);
const dataSize = interleaved.byteLength;
const headerSize = 44;
const totalSize = headerSize + dataSize;
const buffer2 = new ArrayBuffer(totalSize);
const view = new DataView(buffer2);
// RIFF header
writeString(view, 0, "RIFF");
view.setUint32(4, totalSize - 8, true);
writeString(view, 8, "WAVE");
// fmt sub-chunk
writeString(view, 12, "fmt ");
view.setUint32(16, 16, true); // sub-chunk size
view.setUint16(20, format, true);
view.setUint16(22, numChannels, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, byteRate, true);
view.setUint16(32, blockAlign, true);
view.setUint16(34, bitsPerSample, true);
// data sub-chunk
writeString(view, 36, "data");
view.setUint32(40, dataSize, true);
new Int16Array(buffer2, headerSize).set(interleaved);
return new Blob([buffer2], { type: "audio/wav" });
}
function writeString(view: DataView, offset: number, str: string): void {
for (let i = 0; i < str.length; i++) {
view.setUint8(offset + i, str.charCodeAt(i));
}
}
function transcriptionErrorKey(error: unknown): VoiceRecorderErrorKey {
const detail = error instanceof Error ? error.message : "";
if (detail === "not_configured") return "notConfigured";