refactor(app): replace voice hooks with voice runtime and audio engine architecture
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,548 @@
|
||||
import { getTauri } from "@/utils/tauri";
|
||||
import {
|
||||
loadThinkingToneArrayBuffer,
|
||||
THINKING_TONE_REPEAT_GAP_MS,
|
||||
} from "@/utils/thinking-tone";
|
||||
import type {
|
||||
AudioEngine,
|
||||
AudioEngineCallbacks,
|
||||
AudioPlaybackSource,
|
||||
} from "@/voice/audio-engine-types";
|
||||
|
||||
interface QueuedAudio {
|
||||
audio: AudioPlaybackSource;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
function getAudioContextCtor(): (typeof AudioContext) | null {
|
||||
if (typeof window === "undefined") {
|
||||
return null;
|
||||
}
|
||||
const browserWindow = window as typeof window & {
|
||||
webkitAudioContext?: typeof AudioContext;
|
||||
};
|
||||
return browserWindow.AudioContext ?? browserWindow.webkitAudioContext ?? null;
|
||||
}
|
||||
|
||||
function floatToInt16(sample: number): number {
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
|
||||
}
|
||||
|
||||
function resampleToPcm16(
|
||||
input: Float32Array,
|
||||
inputRate: number,
|
||||
outputRate: number
|
||||
): Uint8Array {
|
||||
if (input.length === 0) {
|
||||
return new Uint8Array(0);
|
||||
}
|
||||
|
||||
const ratio = inputRate / outputRate;
|
||||
const outputLength = Math.max(1, Math.round(input.length / ratio));
|
||||
const output = new Int16Array(outputLength);
|
||||
for (let i = 0; i < outputLength; i += 1) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const i1 = Math.min(input.length - 1, i0 + 1);
|
||||
const frac = sourceIndex - i0;
|
||||
const sample = input[i0]! * (1 - frac) + input[i1]! * frac;
|
||||
output[i] = floatToInt16(sample);
|
||||
}
|
||||
|
||||
return new Uint8Array(output.buffer, output.byteOffset, output.byteLength);
|
||||
}
|
||||
|
||||
function parsePcmSampleRate(mimeType: string): number | null {
|
||||
const match = /rate=(\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
}
|
||||
|
||||
function pcm16LeToAudioBuffer(
|
||||
context: AudioContext,
|
||||
bytes: Uint8Array,
|
||||
sampleRate: number
|
||||
): AudioBuffer {
|
||||
const sampleCount = Math.floor(bytes.length / 2);
|
||||
const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
|
||||
const channel = audioBuffer.getChannelData(0);
|
||||
for (let i = 0; i < sampleCount; i += 1) {
|
||||
const lo = bytes[i * 2]!;
|
||||
const hi = bytes[i * 2 + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value -= 0x10000;
|
||||
}
|
||||
channel[i] = value / 0x8000;
|
||||
}
|
||||
return audioBuffer;
|
||||
}
|
||||
|
||||
async function decodeAudioData(
|
||||
context: AudioContext,
|
||||
buffer: ArrayBuffer
|
||||
): Promise<AudioBuffer> {
|
||||
const maybePromise = context.decodeAudioData(buffer.slice(0));
|
||||
if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
|
||||
return maybePromise as Promise<AudioBuffer>;
|
||||
}
|
||||
return await new Promise<AudioBuffer>((resolve, reject) => {
|
||||
context.decodeAudioData(buffer.slice(0), resolve, reject);
|
||||
});
|
||||
}
|
||||
|
||||
export function createAudioEngine(
|
||||
callbacks: AudioEngineCallbacks,
|
||||
_options?: { traceLabel?: string }
|
||||
): AudioEngine {
|
||||
const refs: {
|
||||
playbackContext: AudioContext | null;
|
||||
captureContext: AudioContext | null;
|
||||
stream: MediaStream | null;
|
||||
source: MediaStreamAudioSourceNode | null;
|
||||
processor: ScriptProcessorNode | null;
|
||||
gain: GainNode | null;
|
||||
started: boolean;
|
||||
muted: boolean;
|
||||
queue: QueuedAudio[];
|
||||
processingQueue: boolean;
|
||||
activePlayback: {
|
||||
source: AudioBufferSourceNode;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
settled: boolean;
|
||||
} | null;
|
||||
looping: {
|
||||
token: number;
|
||||
source: AudioBufferSourceNode | null;
|
||||
gain: GainNode | null;
|
||||
timeout: ReturnType<typeof setTimeout> | null;
|
||||
};
|
||||
thinkingTone: {
|
||||
bytes: Uint8Array | null;
|
||||
buffer: AudioBuffer | null;
|
||||
};
|
||||
} = {
|
||||
playbackContext: null,
|
||||
captureContext: null,
|
||||
stream: null,
|
||||
source: null,
|
||||
processor: null,
|
||||
gain: null,
|
||||
started: false,
|
||||
muted: false,
|
||||
queue: [],
|
||||
processingQueue: false,
|
||||
activePlayback: null,
|
||||
looping: {
|
||||
token: 0,
|
||||
source: null,
|
||||
gain: null,
|
||||
timeout: null,
|
||||
},
|
||||
thinkingTone: {
|
||||
bytes: null,
|
||||
buffer: null,
|
||||
},
|
||||
};
|
||||
|
||||
async function ensurePlaybackContext(): Promise<AudioContext> {
|
||||
if (refs.playbackContext) {
|
||||
if (refs.playbackContext.state === "suspended") {
|
||||
await refs.playbackContext.resume().catch(() => undefined);
|
||||
}
|
||||
return refs.playbackContext;
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
if (context.state === "suspended") {
|
||||
await context.resume().catch(() => undefined);
|
||||
}
|
||||
refs.playbackContext = context;
|
||||
return context;
|
||||
}
|
||||
|
||||
async function ensureCaptureContext(): Promise<AudioContext> {
|
||||
if (refs.captureContext) {
|
||||
if (refs.captureContext.state === "suspended") {
|
||||
await refs.captureContext.resume().catch(() => undefined);
|
||||
}
|
||||
return refs.captureContext;
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
if (context.state === "suspended") {
|
||||
await context.resume().catch(() => undefined);
|
||||
}
|
||||
refs.captureContext = context;
|
||||
return context;
|
||||
}
|
||||
|
||||
async function ensureThinkingToneBytes(): Promise<Uint8Array> {
|
||||
if (refs.thinkingTone.bytes) {
|
||||
return refs.thinkingTone.bytes;
|
||||
}
|
||||
const wav = await loadThinkingToneArrayBuffer();
|
||||
refs.thinkingTone.bytes = new Uint8Array(wav);
|
||||
return refs.thinkingTone.bytes;
|
||||
}
|
||||
|
||||
async function ensureThinkingToneBuffer(): Promise<AudioBuffer> {
|
||||
if (refs.thinkingTone.buffer) {
|
||||
return refs.thinkingTone.buffer;
|
||||
}
|
||||
const context = await ensurePlaybackContext();
|
||||
const wav = await loadThinkingToneArrayBuffer();
|
||||
const audioBuffer = await decodeAudioData(context, wav);
|
||||
refs.thinkingTone.buffer = audioBuffer;
|
||||
return audioBuffer;
|
||||
}
|
||||
|
||||
function stopLooping(): void {
|
||||
refs.looping.token += 1;
|
||||
if (refs.looping.timeout) {
|
||||
clearTimeout(refs.looping.timeout);
|
||||
refs.looping.timeout = null;
|
||||
}
|
||||
if (refs.looping.source) {
|
||||
try {
|
||||
refs.looping.source.onended = null;
|
||||
refs.looping.source.stop();
|
||||
} catch {
|
||||
// Ignore best-effort stop errors.
|
||||
}
|
||||
refs.looping.source.disconnect();
|
||||
refs.looping.source = null;
|
||||
}
|
||||
if (refs.looping.gain) {
|
||||
refs.looping.gain.disconnect();
|
||||
refs.looping.gain = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
|
||||
const context = await ensurePlaybackContext();
|
||||
const arrayBuffer = await audio.arrayBuffer();
|
||||
const type = (audio.type || "").toLowerCase();
|
||||
const audioBuffer = type.startsWith("audio/pcm")
|
||||
? pcm16LeToAudioBuffer(
|
||||
context,
|
||||
new Uint8Array(arrayBuffer),
|
||||
parsePcmSampleRate(type) ?? 24000
|
||||
)
|
||||
: await decodeAudioData(context, arrayBuffer);
|
||||
|
||||
const durationSec = audioBuffer.duration;
|
||||
const source = context.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(context.destination);
|
||||
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
refs.activePlayback = { source, resolve, reject, settled: false };
|
||||
|
||||
const settle = (fn: () => void) => {
|
||||
const active = refs.activePlayback;
|
||||
if (!active || active.source !== source || active.settled) {
|
||||
return;
|
||||
}
|
||||
active.settled = true;
|
||||
refs.activePlayback = null;
|
||||
fn();
|
||||
};
|
||||
|
||||
source.onended = () => {
|
||||
settle(() => resolve(durationSec));
|
||||
};
|
||||
|
||||
try {
|
||||
source.start();
|
||||
} catch (error) {
|
||||
settle(() =>
|
||||
reject(error instanceof Error ? error : new Error(String(error)))
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (refs.processingQueue || refs.queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
refs.processingQueue = true;
|
||||
while (refs.queue.length > 0) {
|
||||
const item = refs.queue.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audio);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
}
|
||||
|
||||
async function stopCapture(): Promise<void> {
|
||||
refs.started = false;
|
||||
|
||||
try {
|
||||
refs.processor?.disconnect();
|
||||
refs.source?.disconnect();
|
||||
refs.gain?.disconnect();
|
||||
} catch {
|
||||
// Ignore best-effort teardown errors.
|
||||
}
|
||||
|
||||
if (refs.stream) {
|
||||
for (const track of refs.stream.getTracks()) {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
// Ignore best-effort teardown errors.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
refs.stream = null;
|
||||
refs.source = null;
|
||||
refs.processor = null;
|
||||
refs.gain = null;
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
|
||||
const captureContext = refs.captureContext;
|
||||
refs.captureContext = null;
|
||||
if (captureContext && captureContext.state !== "closed") {
|
||||
await captureContext.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async initialize() {
|
||||
await ensurePlaybackContext();
|
||||
await Promise.all([ensureThinkingToneBytes(), ensureThinkingToneBuffer()]);
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
stopLooping();
|
||||
this.stop();
|
||||
this.clearQueue();
|
||||
await stopCapture();
|
||||
|
||||
const playbackContext = refs.playbackContext;
|
||||
refs.playbackContext = null;
|
||||
refs.thinkingTone.buffer = null;
|
||||
if (playbackContext && playbackContext.state !== "closed") {
|
||||
await playbackContext.close().catch(() => undefined);
|
||||
}
|
||||
},
|
||||
|
||||
async startCapture() {
|
||||
if (refs.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const missingNavigator =
|
||||
typeof navigator === "undefined" ||
|
||||
!navigator.mediaDevices ||
|
||||
typeof navigator.mediaDevices.getUserMedia !== "function";
|
||||
const secureContext =
|
||||
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
|
||||
? window.isSecureContext
|
||||
: true;
|
||||
const currentOrigin =
|
||||
typeof window !== "undefined" && window.location
|
||||
? window.location.origin
|
||||
: "unknown";
|
||||
const isTauri = getTauri() !== null;
|
||||
|
||||
if (missingNavigator) {
|
||||
throw new Error("Microphone capture is not supported in this environment");
|
||||
}
|
||||
if (!secureContext && !isTauri) {
|
||||
throw new Error(
|
||||
`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const context = await ensureCaptureContext();
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const processor = context.createScriptProcessor(4096, 1, 1);
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 0;
|
||||
|
||||
processor.onaudioprocess = (event) => {
|
||||
if (!refs.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < input.length; i += 1) {
|
||||
const sample = input[i]!;
|
||||
sumSquares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||
const normalized = Math.min(1, Math.max(0, rms * 2));
|
||||
callbacks.onVolumeLevel(normalized);
|
||||
|
||||
if (refs.muted) {
|
||||
return;
|
||||
}
|
||||
|
||||
callbacks.onCaptureData(resampleToPcm16(input, context.sampleRate, 16000));
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(gain);
|
||||
gain.connect(context.destination);
|
||||
|
||||
refs.started = true;
|
||||
refs.stream = stream;
|
||||
refs.source = source;
|
||||
refs.processor = processor;
|
||||
refs.gain = gain;
|
||||
} catch (error) {
|
||||
await stopCapture();
|
||||
const wrapped = error instanceof Error ? error : new Error(String(error));
|
||||
callbacks.onError?.(wrapped);
|
||||
throw wrapped;
|
||||
}
|
||||
},
|
||||
|
||||
async stopCapture() {
|
||||
await stopCapture();
|
||||
},
|
||||
|
||||
toggleMute() {
|
||||
refs.muted = !refs.muted;
|
||||
if (refs.muted) {
|
||||
callbacks.onVolumeLevel(0);
|
||||
}
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
isMuted() {
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
async play(audio: AudioPlaybackSource) {
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
refs.queue.push({ audio, resolve, reject });
|
||||
if (!refs.processingQueue) {
|
||||
void processQueue();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
stop() {
|
||||
if (refs.activePlayback) {
|
||||
const active = refs.activePlayback;
|
||||
refs.activePlayback = null;
|
||||
try {
|
||||
active.source.onended = null;
|
||||
active.source.stop();
|
||||
} catch {
|
||||
// Ignore best-effort stop errors.
|
||||
}
|
||||
if (!active.settled) {
|
||||
active.settled = true;
|
||||
active.reject(new Error("Playback stopped"));
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
clearQueue() {
|
||||
while (refs.queue.length > 0) {
|
||||
refs.queue.shift()!.reject(new Error("Playback stopped"));
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
},
|
||||
|
||||
isPlaying() {
|
||||
return refs.activePlayback !== null;
|
||||
},
|
||||
|
||||
playLooping(audio, gapMs) {
|
||||
if (refs.looping.source || refs.looping.timeout) {
|
||||
return;
|
||||
}
|
||||
|
||||
const token = refs.looping.token + 1;
|
||||
refs.looping.token = token;
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const context = await ensurePlaybackContext();
|
||||
const audioBuffer =
|
||||
audio.byteLength > 0
|
||||
? pcm16LeToAudioBuffer(context, audio, 16000)
|
||||
: await ensureThinkingToneBuffer();
|
||||
if (refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 1;
|
||||
gain.connect(context.destination);
|
||||
refs.looping.gain = gain;
|
||||
|
||||
const playOnce = () => {
|
||||
if (refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
const source = context.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(gain);
|
||||
refs.looping.source = source;
|
||||
source.onended = () => {
|
||||
if (refs.looping.source === source) {
|
||||
refs.looping.source = null;
|
||||
}
|
||||
if (refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
refs.looping.timeout = setTimeout(
|
||||
playOnce,
|
||||
gapMs || THINKING_TONE_REPEAT_GAP_MS
|
||||
);
|
||||
};
|
||||
source.start();
|
||||
};
|
||||
|
||||
playOnce();
|
||||
} catch (error) {
|
||||
callbacks.onError?.(
|
||||
error instanceof Error ? error : new Error(String(error))
|
||||
);
|
||||
}
|
||||
})();
|
||||
},
|
||||
|
||||
stopLooping,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user