fix(webui): convert WebM to WAV for Xiaomi MiMo ASR transcription
MiMo ASR (mimo-v2.5-asr) only accepts audio/wav, audio/mp3, and audio/mpeg formats. Web browsers record in WebM/Opus by default, causing the API to reject the payload with a transcription error. This change adds a frontend WebM→WAV converter using the Web Audio API (DecodeAudioData + PCM encoding) that activates only when the configured transcription provider is 'xiaomi_mimo'. Other providers are unaffected — they continue to receive the original browser format. Tested and confirmed working on WebUI.
This commit is contained in:
@@ -172,6 +172,7 @@ interface ThreadComposerProps {
|
||||
workspaceError?: string | null;
|
||||
onWorkspaceScopeChange?: (scope: WorkspaceScopePayload) => void;
|
||||
pendingQueueKey?: string | null;
|
||||
transcriptionProvider?: string | null;
|
||||
}
|
||||
|
||||
const COMMAND_ICONS: Record<string, LucideIcon> = {
|
||||
@@ -782,6 +783,7 @@ export function ThreadComposer({
|
||||
workspaceError = null,
|
||||
onWorkspaceScopeChange,
|
||||
pendingQueueKey = null,
|
||||
transcriptionProvider = null,
|
||||
}: ThreadComposerProps) {
|
||||
const { t } = useTranslation();
|
||||
const [value, setValue] = useState("");
|
||||
@@ -1193,6 +1195,7 @@ export function ThreadComposer({
|
||||
onError: setVoiceError,
|
||||
onTranscript: appendTranscription,
|
||||
onTranscribeAudio,
|
||||
wantsWav: transcriptionProvider === "xiaomi_mimo",
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
|
||||
@@ -736,6 +736,7 @@ export function ThreadShell({
|
||||
workspaceError={workspaceError}
|
||||
onWorkspaceScopeChange={onWorkspaceScopeChange}
|
||||
pendingQueueKey={chatId}
|
||||
transcriptionProvider={settingsSnapshot?.transcription?.provider}
|
||||
/>
|
||||
) : (
|
||||
<ThreadComposer
|
||||
@@ -765,6 +766,7 @@ export function ThreadShell({
|
||||
workspaceScopeDisabled={workspaceScopeDisabled}
|
||||
workspaceError={workspaceError}
|
||||
onWorkspaceScopeChange={onWorkspaceScopeChange}
|
||||
transcriptionProvider={settingsSnapshot?.transcription?.provider}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
|
||||
@@ -42,6 +42,8 @@ interface VoiceRecorderOptions {
|
||||
onError: (key: VoiceRecorderErrorKey) => void;
|
||||
onTranscript: (text: string) => void;
|
||||
onTranscribeAudio?: (dataUrl: string, options?: { durationMs?: number }) => Promise<string>;
|
||||
/** When true, convert recorded audio to WAV before sending (needed for providers that don't support WebM). */
|
||||
wantsWav?: boolean;
|
||||
}
|
||||
|
||||
export function useVoiceRecorder({
|
||||
@@ -50,6 +52,7 @@ export function useVoiceRecorder({
|
||||
onError,
|
||||
onTranscript,
|
||||
onTranscribeAudio,
|
||||
wantsWav = false,
|
||||
}: VoiceRecorderOptions) {
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const chunksRef = useRef<BlobPart[]>([]);
|
||||
@@ -223,7 +226,9 @@ export function useVoiceRecorder({
|
||||
return;
|
||||
}
|
||||
setState("transcribing");
|
||||
void blobToDataUrl(new Blob(chunks, { type: mimeType }))
|
||||
const blob = new Blob(chunks, { type: mimeType });
|
||||
const audioPromise = wantsWav ? convertBlobToWav(blob) : blobToDataUrl(blob);
|
||||
void audioPromise
|
||||
.then((dataUrl) => onTranscribeAudio(dataUrl, { durationMs }))
|
||||
.then(onTranscript)
|
||||
.catch((error) => onError(transcriptionErrorKey(error)))
|
||||
@@ -260,6 +265,7 @@ export function useVoiceRecorder({
|
||||
startWaveform,
|
||||
state,
|
||||
stopRecording,
|
||||
wantsWav,
|
||||
]);
|
||||
|
||||
const startRecordingWithDeferredStop = useCallback(() => {
|
||||
@@ -414,6 +420,90 @@ function blobToDataUrl(blob: Blob): Promise<string> {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert any browser-recorded audio blob (typically webm/opus) to WAV
|
||||
* using the Web Audio API. This avoids sending unsupported formats
|
||||
* (e.g. webm) to ASR providers that only accept wav/mp3/mpeg.
|
||||
*/
|
||||
async function convertBlobToWav(blob: Blob): Promise<string> {
|
||||
const AudioCtx = audioContextConstructor();
|
||||
if (!AudioCtx) return blobToDataUrl(blob);
|
||||
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
const ctx = new AudioCtx();
|
||||
try {
|
||||
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
|
||||
const wavBlob = audioBufferToWav(audioBuffer);
|
||||
return blobToDataUrl(wavBlob);
|
||||
} finally {
|
||||
void ctx.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode an AudioBuffer as a 16-bit PCM WAV Blob.
|
||||
*/
|
||||
function audioBufferToWav(buffer: AudioBuffer): Blob {
|
||||
const numChannels = buffer.numberOfChannels;
|
||||
const sampleRate = buffer.sampleRate;
|
||||
const format = 1; // PCM
|
||||
const bitsPerSample = 16;
|
||||
|
||||
// Interleave channels
|
||||
const channels: Float32Array[] = [];
|
||||
for (let ch = 0; ch < numChannels; ch++) {
|
||||
channels.push(buffer.getChannelData(ch));
|
||||
}
|
||||
const length = channels[0].length;
|
||||
const interleaved = new Int16Array(length * numChannels);
|
||||
for (let i = 0; i < length; i++) {
|
||||
for (let ch = 0; ch < numChannels; ch++) {
|
||||
const sample = Math.max(-1, Math.min(1, channels[ch][i]));
|
||||
interleaved[i * numChannels + ch] = sample < 0
|
||||
? sample * 0x8000
|
||||
: sample * 0x7FFF;
|
||||
}
|
||||
}
|
||||
|
||||
const byteRate = sampleRate * numChannels * (bitsPerSample / 8);
|
||||
const blockAlign = numChannels * (bitsPerSample / 8);
|
||||
const dataSize = interleaved.byteLength;
|
||||
const headerSize = 44;
|
||||
const totalSize = headerSize + dataSize;
|
||||
|
||||
const buffer2 = new ArrayBuffer(totalSize);
|
||||
const view = new DataView(buffer2);
|
||||
|
||||
// RIFF header
|
||||
writeString(view, 0, "RIFF");
|
||||
view.setUint32(4, totalSize - 8, true);
|
||||
writeString(view, 8, "WAVE");
|
||||
|
||||
// fmt sub-chunk
|
||||
writeString(view, 12, "fmt ");
|
||||
view.setUint32(16, 16, true); // sub-chunk size
|
||||
view.setUint16(20, format, true);
|
||||
view.setUint16(22, numChannels, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, byteRate, true);
|
||||
view.setUint16(32, blockAlign, true);
|
||||
view.setUint16(34, bitsPerSample, true);
|
||||
|
||||
// data sub-chunk
|
||||
writeString(view, 36, "data");
|
||||
view.setUint32(40, dataSize, true);
|
||||
|
||||
new Int16Array(buffer2, headerSize).set(interleaved);
|
||||
|
||||
return new Blob([buffer2], { type: "audio/wav" });
|
||||
}
|
||||
|
||||
function writeString(view: DataView, offset: number, str: string): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
function transcriptionErrorKey(error: unknown): VoiceRecorderErrorKey {
|
||||
const detail = error instanceof Error ? error.message : "";
|
||||
if (detail === "not_configured") return "notConfigured";
|
||||
|
||||
Reference in New Issue
Block a user