mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-08-06 17:38:35 +00:00
fix(webui): convert WebM to WAV for Xiaomi MiMo ASR transcription
MiMo ASR (mimo-v2.5-asr) only accepts audio/wav, audio/mp3, and audio/mpeg formats. Web browsers record in WebM/Opus by default, causing the API to reject the payload with a transcription error. This change adds a frontend WebM→WAV converter using the Web Audio API (DecodeAudioData + PCM encoding) that activates only when the configured transcription provider is 'xiaomi_mimo'. Other providers are unaffected — they continue to receive the original browser format. Tested and confirmed working on WebUI.
This commit is contained in:
parent
7899857201
commit
28c8c89a42
@ -172,6 +172,7 @@ interface ThreadComposerProps {
|
|||||||
workspaceError?: string | null;
|
workspaceError?: string | null;
|
||||||
onWorkspaceScopeChange?: (scope: WorkspaceScopePayload) => void;
|
onWorkspaceScopeChange?: (scope: WorkspaceScopePayload) => void;
|
||||||
pendingQueueKey?: string | null;
|
pendingQueueKey?: string | null;
|
||||||
|
transcriptionProvider?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const COMMAND_ICONS: Record<string, LucideIcon> = {
|
const COMMAND_ICONS: Record<string, LucideIcon> = {
|
||||||
@ -782,6 +783,7 @@ export function ThreadComposer({
|
|||||||
workspaceError = null,
|
workspaceError = null,
|
||||||
onWorkspaceScopeChange,
|
onWorkspaceScopeChange,
|
||||||
pendingQueueKey = null,
|
pendingQueueKey = null,
|
||||||
|
transcriptionProvider = null,
|
||||||
}: ThreadComposerProps) {
|
}: ThreadComposerProps) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
const [value, setValue] = useState("");
|
const [value, setValue] = useState("");
|
||||||
@ -1193,6 +1195,7 @@ export function ThreadComposer({
|
|||||||
onError: setVoiceError,
|
onError: setVoiceError,
|
||||||
onTranscript: appendTranscription,
|
onTranscript: appendTranscription,
|
||||||
onTranscribeAudio,
|
onTranscribeAudio,
|
||||||
|
wantsWav: transcriptionProvider === "xiaomi_mimo",
|
||||||
});
|
});
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|||||||
@ -736,6 +736,7 @@ export function ThreadShell({
|
|||||||
workspaceError={workspaceError}
|
workspaceError={workspaceError}
|
||||||
onWorkspaceScopeChange={onWorkspaceScopeChange}
|
onWorkspaceScopeChange={onWorkspaceScopeChange}
|
||||||
pendingQueueKey={chatId}
|
pendingQueueKey={chatId}
|
||||||
|
transcriptionProvider={settingsSnapshot?.transcription?.provider}
|
||||||
/>
|
/>
|
||||||
) : (
|
) : (
|
||||||
<ThreadComposer
|
<ThreadComposer
|
||||||
@ -765,6 +766,7 @@ export function ThreadShell({
|
|||||||
workspaceScopeDisabled={workspaceScopeDisabled}
|
workspaceScopeDisabled={workspaceScopeDisabled}
|
||||||
workspaceError={workspaceError}
|
workspaceError={workspaceError}
|
||||||
onWorkspaceScopeChange={onWorkspaceScopeChange}
|
onWorkspaceScopeChange={onWorkspaceScopeChange}
|
||||||
|
transcriptionProvider={settingsSnapshot?.transcription?.provider}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
</>
|
</>
|
||||||
|
|||||||
@ -42,6 +42,8 @@ interface VoiceRecorderOptions {
|
|||||||
onError: (key: VoiceRecorderErrorKey) => void;
|
onError: (key: VoiceRecorderErrorKey) => void;
|
||||||
onTranscript: (text: string) => void;
|
onTranscript: (text: string) => void;
|
||||||
onTranscribeAudio?: (dataUrl: string, options?: { durationMs?: number }) => Promise<string>;
|
onTranscribeAudio?: (dataUrl: string, options?: { durationMs?: number }) => Promise<string>;
|
||||||
|
/** When true, convert recorded audio to WAV before sending (needed for providers that don't support WebM). */
|
||||||
|
wantsWav?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useVoiceRecorder({
|
export function useVoiceRecorder({
|
||||||
@ -50,6 +52,7 @@ export function useVoiceRecorder({
|
|||||||
onError,
|
onError,
|
||||||
onTranscript,
|
onTranscript,
|
||||||
onTranscribeAudio,
|
onTranscribeAudio,
|
||||||
|
wantsWav = false,
|
||||||
}: VoiceRecorderOptions) {
|
}: VoiceRecorderOptions) {
|
||||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||||
const chunksRef = useRef<BlobPart[]>([]);
|
const chunksRef = useRef<BlobPart[]>([]);
|
||||||
@ -223,7 +226,9 @@ export function useVoiceRecorder({
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
setState("transcribing");
|
setState("transcribing");
|
||||||
void blobToDataUrl(new Blob(chunks, { type: mimeType }))
|
const blob = new Blob(chunks, { type: mimeType });
|
||||||
|
const audioPromise = wantsWav ? convertBlobToWav(blob) : blobToDataUrl(blob);
|
||||||
|
void audioPromise
|
||||||
.then((dataUrl) => onTranscribeAudio(dataUrl, { durationMs }))
|
.then((dataUrl) => onTranscribeAudio(dataUrl, { durationMs }))
|
||||||
.then(onTranscript)
|
.then(onTranscript)
|
||||||
.catch((error) => onError(transcriptionErrorKey(error)))
|
.catch((error) => onError(transcriptionErrorKey(error)))
|
||||||
@ -260,6 +265,7 @@ export function useVoiceRecorder({
|
|||||||
startWaveform,
|
startWaveform,
|
||||||
state,
|
state,
|
||||||
stopRecording,
|
stopRecording,
|
||||||
|
wantsWav,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const startRecordingWithDeferredStop = useCallback(() => {
|
const startRecordingWithDeferredStop = useCallback(() => {
|
||||||
@ -414,6 +420,90 @@ function blobToDataUrl(blob: Blob): Promise<string> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert any browser-recorded audio blob (typically webm/opus) to WAV
|
||||||
|
* using the Web Audio API. This avoids sending unsupported formats
|
||||||
|
* (e.g. webm) to ASR providers that only accept wav/mp3/mpeg.
|
||||||
|
*/
|
||||||
|
async function convertBlobToWav(blob: Blob): Promise<string> {
|
||||||
|
const AudioCtx = audioContextConstructor();
|
||||||
|
if (!AudioCtx) return blobToDataUrl(blob);
|
||||||
|
|
||||||
|
const arrayBuffer = await blob.arrayBuffer();
|
||||||
|
const ctx = new AudioCtx();
|
||||||
|
try {
|
||||||
|
const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
|
||||||
|
const wavBlob = audioBufferToWav(audioBuffer);
|
||||||
|
return blobToDataUrl(wavBlob);
|
||||||
|
} finally {
|
||||||
|
void ctx.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Encode an AudioBuffer as a 16-bit PCM WAV Blob.
|
||||||
|
*/
|
||||||
|
function audioBufferToWav(buffer: AudioBuffer): Blob {
|
||||||
|
const numChannels = buffer.numberOfChannels;
|
||||||
|
const sampleRate = buffer.sampleRate;
|
||||||
|
const format = 1; // PCM
|
||||||
|
const bitsPerSample = 16;
|
||||||
|
|
||||||
|
// Interleave channels
|
||||||
|
const channels: Float32Array[] = [];
|
||||||
|
for (let ch = 0; ch < numChannels; ch++) {
|
||||||
|
channels.push(buffer.getChannelData(ch));
|
||||||
|
}
|
||||||
|
const length = channels[0].length;
|
||||||
|
const interleaved = new Int16Array(length * numChannels);
|
||||||
|
for (let i = 0; i < length; i++) {
|
||||||
|
for (let ch = 0; ch < numChannels; ch++) {
|
||||||
|
const sample = Math.max(-1, Math.min(1, channels[ch][i]));
|
||||||
|
interleaved[i * numChannels + ch] = sample < 0
|
||||||
|
? sample * 0x8000
|
||||||
|
: sample * 0x7FFF;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const byteRate = sampleRate * numChannels * (bitsPerSample / 8);
|
||||||
|
const blockAlign = numChannels * (bitsPerSample / 8);
|
||||||
|
const dataSize = interleaved.byteLength;
|
||||||
|
const headerSize = 44;
|
||||||
|
const totalSize = headerSize + dataSize;
|
||||||
|
|
||||||
|
const buffer2 = new ArrayBuffer(totalSize);
|
||||||
|
const view = new DataView(buffer2);
|
||||||
|
|
||||||
|
// RIFF header
|
||||||
|
writeString(view, 0, "RIFF");
|
||||||
|
view.setUint32(4, totalSize - 8, true);
|
||||||
|
writeString(view, 8, "WAVE");
|
||||||
|
|
||||||
|
// fmt sub-chunk
|
||||||
|
writeString(view, 12, "fmt ");
|
||||||
|
view.setUint32(16, 16, true); // sub-chunk size
|
||||||
|
view.setUint16(20, format, true);
|
||||||
|
view.setUint16(22, numChannels, true);
|
||||||
|
view.setUint32(24, sampleRate, true);
|
||||||
|
view.setUint32(28, byteRate, true);
|
||||||
|
view.setUint16(32, blockAlign, true);
|
||||||
|
view.setUint16(34, bitsPerSample, true);
|
||||||
|
|
||||||
|
// data sub-chunk
|
||||||
|
writeString(view, 36, "data");
|
||||||
|
view.setUint32(40, dataSize, true);
|
||||||
|
|
||||||
|
new Int16Array(buffer2, headerSize).set(interleaved);
|
||||||
|
|
||||||
|
return new Blob([buffer2], { type: "audio/wav" });
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeString(view: DataView, offset: number, str: string): void {
|
||||||
|
for (let i = 0; i < str.length; i++) {
|
||||||
|
view.setUint8(offset + i, str.charCodeAt(i));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function transcriptionErrorKey(error: unknown): VoiceRecorderErrorKey {
|
function transcriptionErrorKey(error: unknown): VoiceRecorderErrorKey {
|
||||||
const detail = error instanceof Error ? error.message : "";
|
const detail = error instanceof Error ? error.message : "";
|
||||||
if (detail === "not_configured") return "notConfigured";
|
if (detail === "not_configured") return "notConfigured";
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user