refactor(app): replace voice hooks with voice runtime and audio engine architecture
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,29 @@
|
||||
export interface AudioEngineCallbacks {
|
||||
onCaptureData(pcm: Uint8Array): void;
|
||||
onVolumeLevel(level: number): void;
|
||||
onError?(error: Error): void;
|
||||
}
|
||||
|
||||
export interface AudioPlaybackSource {
|
||||
arrayBuffer(): Promise<ArrayBuffer>;
|
||||
size: number;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export interface AudioEngine {
|
||||
initialize(): Promise<void>;
|
||||
destroy(): Promise<void>;
|
||||
|
||||
startCapture(): Promise<void>;
|
||||
stopCapture(): Promise<void>;
|
||||
toggleMute(): boolean;
|
||||
isMuted(): boolean;
|
||||
|
||||
play(audio: AudioPlaybackSource): Promise<number>;
|
||||
stop(): void;
|
||||
clearQueue(): void;
|
||||
isPlaying(): boolean;
|
||||
|
||||
playLooping(pcm: Uint8Array, gapMs: number): void;
|
||||
stopLooping(): void;
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
export * from "./audio-engine.native";
|
||||
@@ -0,0 +1,640 @@
|
||||
import {
|
||||
addExpoTwoWayAudioEventListener,
|
||||
getMicrophonePermissionsAsync,
|
||||
initialize,
|
||||
playPCMData,
|
||||
requestMicrophonePermissionsAsync,
|
||||
resumePlayback,
|
||||
stopPlayback,
|
||||
tearDown,
|
||||
toggleRecording,
|
||||
} from "@boudra/expo-two-way-audio";
|
||||
import {
|
||||
THINKING_TONE_REPEAT_GAP_MS,
|
||||
} from "@/utils/thinking-tone";
|
||||
import {
|
||||
THINKING_TONE_NATIVE_PCM_BASE64,
|
||||
THINKING_TONE_NATIVE_PCM_DURATION_MS,
|
||||
} from "@/utils/thinking-tone.native-pcm";
|
||||
import type {
|
||||
AudioEngine,
|
||||
AudioEngineCallbacks,
|
||||
AudioPlaybackSource,
|
||||
} from "@/voice/audio-engine-types";
|
||||
import { Buffer } from "buffer";
|
||||
|
||||
interface QueuedAudio {
|
||||
audio: AudioPlaybackSource;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
interface CuePcm {
|
||||
pcm16k: Uint8Array;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
interface StreamDebugState {
|
||||
count: number;
|
||||
totalValue: number;
|
||||
maxValue: number;
|
||||
startedAtMs: number;
|
||||
lastAtMs: number;
|
||||
maxGapMs: number;
|
||||
nextLogAtMs: number;
|
||||
}
|
||||
|
||||
interface AudioEngineTraceOptions {
|
||||
traceLabel?: string;
|
||||
}
|
||||
|
||||
let nextAudioEngineInstanceId = 1;
|
||||
|
||||
function getTraceStack(): string | undefined {
|
||||
const stack = new Error().stack;
|
||||
if (!stack) {
|
||||
return undefined;
|
||||
}
|
||||
return stack
|
||||
.split("\n")
|
||||
.slice(2, 7)
|
||||
.map((line) => line.trim())
|
||||
.join(" | ");
|
||||
}
|
||||
|
||||
function resamplePcm16(
|
||||
pcm: Uint8Array,
|
||||
fromRate: number,
|
||||
toRate: number
|
||||
): Uint8Array {
|
||||
if (fromRate === toRate) {
|
||||
return pcm;
|
||||
}
|
||||
|
||||
const inputSamples = Math.floor(pcm.length / 2);
|
||||
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
|
||||
const output = new Uint8Array(outputSamples * 2);
|
||||
const ratio = fromRate / toRate;
|
||||
|
||||
const readInt16 = (sampleIndex: number): number => {
|
||||
const offset = sampleIndex * 2;
|
||||
if (offset + 1 >= pcm.length) {
|
||||
return 0;
|
||||
}
|
||||
const lo = pcm[offset]!;
|
||||
const hi = pcm[offset + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value -= 0x10000;
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const writeInt16 = (sampleIndex: number, value: number): void => {
|
||||
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
|
||||
const offset = sampleIndex * 2;
|
||||
output[offset] = clamped & 0xff;
|
||||
output[offset + 1] = (clamped >> 8) & 0xff;
|
||||
};
|
||||
|
||||
for (let i = 0; i < outputSamples; i += 1) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const frac = sourceIndex - i0;
|
||||
const s0 = readInt16(i0);
|
||||
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
|
||||
writeInt16(i, s0 + (s1 - s0) * frac);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
function parsePcmSampleRate(mimeType: string): number | null {
|
||||
const match = /rate=(\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
}
|
||||
|
||||
function logVoiceNative(event: string, details?: Record<string, unknown>): void {
|
||||
if (details) {
|
||||
console.log(`[VoiceNative] ${event}`, details);
|
||||
return;
|
||||
}
|
||||
console.log(`[VoiceNative] ${event}`);
|
||||
}
|
||||
|
||||
function createStreamDebugState(nowMs: number): StreamDebugState {
|
||||
return {
|
||||
count: 0,
|
||||
totalValue: 0,
|
||||
maxValue: 0,
|
||||
startedAtMs: nowMs,
|
||||
lastAtMs: nowMs,
|
||||
maxGapMs: 0,
|
||||
nextLogAtMs: nowMs + 1000,
|
||||
};
|
||||
}
|
||||
|
||||
export function createAudioEngine(
|
||||
callbacks: AudioEngineCallbacks,
|
||||
options?: AudioEngineTraceOptions
|
||||
): AudioEngine {
|
||||
const engineId = nextAudioEngineInstanceId++;
|
||||
const traceLabel = options?.traceLabel ?? "unknown";
|
||||
const traceStack = getTraceStack();
|
||||
const refs: {
|
||||
initialized: boolean;
|
||||
captureActive: boolean;
|
||||
muted: boolean;
|
||||
queue: QueuedAudio[];
|
||||
processingQueue: boolean;
|
||||
playbackTimeout: ReturnType<typeof setTimeout> | null;
|
||||
activePlayback: {
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
settled: boolean;
|
||||
} | null;
|
||||
looping: {
|
||||
active: boolean;
|
||||
token: number;
|
||||
timeout: ReturnType<typeof setTimeout> | null;
|
||||
};
|
||||
thinkingTone: CuePcm | null;
|
||||
captureDebug: StreamDebugState | null;
|
||||
volumeDebug: StreamDebugState | null;
|
||||
destroyed: boolean;
|
||||
} = {
|
||||
initialized: false,
|
||||
captureActive: false,
|
||||
muted: false,
|
||||
queue: [],
|
||||
processingQueue: false,
|
||||
playbackTimeout: null,
|
||||
activePlayback: null,
|
||||
looping: {
|
||||
active: false,
|
||||
token: 0,
|
||||
timeout: null,
|
||||
},
|
||||
thinkingTone: null,
|
||||
captureDebug: null,
|
||||
volumeDebug: null,
|
||||
destroyed: false,
|
||||
};
|
||||
|
||||
logVoiceNative("engine_created", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
stack: traceStack,
|
||||
});
|
||||
|
||||
function resetCaptureDebug(): void {
|
||||
refs.captureDebug = null;
|
||||
}
|
||||
|
||||
function resetVolumeDebug(): void {
|
||||
refs.volumeDebug = null;
|
||||
}
|
||||
|
||||
function recordCaptureChunk(byteLength: number): void {
|
||||
const nowMs = Date.now();
|
||||
const debug = refs.captureDebug ?? createStreamDebugState(nowMs);
|
||||
const gapMs = debug.count === 0 ? 0 : nowMs - debug.lastAtMs;
|
||||
debug.count += 1;
|
||||
debug.totalValue += byteLength;
|
||||
debug.maxValue = Math.max(debug.maxValue, byteLength);
|
||||
debug.maxGapMs = Math.max(debug.maxGapMs, gapMs);
|
||||
debug.lastAtMs = nowMs;
|
||||
|
||||
if (nowMs >= debug.nextLogAtMs) {
|
||||
const elapsedMs = Math.max(1, nowMs - debug.startedAtMs);
|
||||
logVoiceNative("capture_summary", {
|
||||
chunks: debug.count,
|
||||
totalBytes: debug.totalValue,
|
||||
averageChunkBytes: Math.round(debug.totalValue / debug.count),
|
||||
maxChunkBytes: debug.maxValue,
|
||||
chunksPerSecond: Number(((debug.count * 1000) / elapsedMs).toFixed(1)),
|
||||
maxGapMs: debug.maxGapMs,
|
||||
muted: refs.muted,
|
||||
});
|
||||
refs.captureDebug = createStreamDebugState(nowMs);
|
||||
return;
|
||||
}
|
||||
|
||||
refs.captureDebug = debug;
|
||||
}
|
||||
|
||||
function recordVolumeLevel(level: number): void {
|
||||
const nowMs = Date.now();
|
||||
const debug = refs.volumeDebug ?? createStreamDebugState(nowMs);
|
||||
const gapMs = debug.count === 0 ? 0 : nowMs - debug.lastAtMs;
|
||||
debug.count += 1;
|
||||
debug.totalValue += level;
|
||||
debug.maxValue = Math.max(debug.maxValue, level);
|
||||
debug.maxGapMs = Math.max(debug.maxGapMs, gapMs);
|
||||
debug.lastAtMs = nowMs;
|
||||
|
||||
if (nowMs >= debug.nextLogAtMs) {
|
||||
const elapsedMs = Math.max(1, nowMs - debug.startedAtMs);
|
||||
logVoiceNative("volume_summary", {
|
||||
samples: debug.count,
|
||||
averageLevel: Number((debug.totalValue / debug.count).toFixed(3)),
|
||||
peakLevel: Number(debug.maxValue.toFixed(3)),
|
||||
samplesPerSecond: Number(((debug.count * 1000) / elapsedMs).toFixed(1)),
|
||||
maxGapMs: debug.maxGapMs,
|
||||
muted: refs.muted,
|
||||
});
|
||||
refs.volumeDebug = createStreamDebugState(nowMs);
|
||||
return;
|
||||
}
|
||||
|
||||
refs.volumeDebug = debug;
|
||||
}
|
||||
|
||||
const microphoneSubscription = addExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
(event) => {
|
||||
console.log("[VoiceNative] onMicrophoneData", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
captureActive: refs.captureActive,
|
||||
muted: refs.muted,
|
||||
bytes: event.data.byteLength,
|
||||
});
|
||||
if (!refs.captureActive || refs.muted) {
|
||||
return;
|
||||
}
|
||||
recordCaptureChunk(event.data.byteLength);
|
||||
callbacks.onCaptureData(event.data);
|
||||
}
|
||||
);
|
||||
logVoiceNative("listener_added", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
event: "onMicrophoneData",
|
||||
});
|
||||
|
||||
const volumeSubscription = addExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
(event) => {
|
||||
console.log("[VoiceNative] onInputVolumeLevelData", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
captureActive: refs.captureActive,
|
||||
muted: refs.muted,
|
||||
level: event.data,
|
||||
});
|
||||
if (!refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
const level = refs.muted ? 0 : event.data;
|
||||
recordVolumeLevel(level);
|
||||
callbacks.onVolumeLevel(level);
|
||||
}
|
||||
);
|
||||
logVoiceNative("listener_added", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
event: "onInputVolumeLevelData",
|
||||
});
|
||||
|
||||
async function ensureInitialized(): Promise<void> {
|
||||
if (refs.initialized) {
|
||||
return;
|
||||
}
|
||||
logVoiceNative("initialize_start");
|
||||
await initialize();
|
||||
refs.initialized = true;
|
||||
logVoiceNative("initialize_complete");
|
||||
}
|
||||
|
||||
async function ensureMicrophonePermission(): Promise<void> {
|
||||
let permission = await getMicrophonePermissionsAsync().catch(() => null);
|
||||
if (!permission?.granted) {
|
||||
permission = await requestMicrophonePermissionsAsync().catch(() => null);
|
||||
}
|
||||
if (!permission?.granted) {
|
||||
throw new Error(
|
||||
"Microphone permission is required to capture audio. Please enable microphone access in system settings."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureThinkingTone(): Promise<CuePcm> {
|
||||
if (refs.thinkingTone) {
|
||||
return refs.thinkingTone;
|
||||
}
|
||||
const pcm16k = Buffer.from(THINKING_TONE_NATIVE_PCM_BASE64, "base64");
|
||||
const durationMs = THINKING_TONE_NATIVE_PCM_DURATION_MS;
|
||||
refs.thinkingTone = { pcm16k, durationMs };
|
||||
logVoiceNative("thinking_tone_loaded", {
|
||||
bytes: pcm16k.byteLength,
|
||||
durationMs,
|
||||
});
|
||||
return refs.thinkingTone;
|
||||
}
|
||||
|
||||
function clearPlaybackTimeout(): void {
|
||||
if (refs.playbackTimeout) {
|
||||
clearTimeout(refs.playbackTimeout);
|
||||
refs.playbackTimeout = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
|
||||
await ensureInitialized();
|
||||
resumePlayback();
|
||||
|
||||
return await new Promise<number>(async (resolve, reject) => {
|
||||
refs.activePlayback = { resolve, reject, settled: false };
|
||||
|
||||
try {
|
||||
const arrayBuffer = await audio.arrayBuffer();
|
||||
const pcm = new Uint8Array(arrayBuffer);
|
||||
const inputRate = parsePcmSampleRate(audio.type || "") ?? 24000;
|
||||
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
|
||||
const durationSec = pcm16k.length / 2 / 16000;
|
||||
logVoiceNative("playback_start", {
|
||||
inputBytes: pcm.byteLength,
|
||||
outputBytes: pcm16k.byteLength,
|
||||
inputRate,
|
||||
durationMs: Math.round(durationSec * 1000),
|
||||
queueLength: refs.queue.length,
|
||||
});
|
||||
|
||||
playPCMData(pcm16k);
|
||||
clearPlaybackTimeout();
|
||||
refs.playbackTimeout = setTimeout(() => {
|
||||
clearPlaybackTimeout();
|
||||
const active = refs.activePlayback;
|
||||
if (!active || active.settled) {
|
||||
return;
|
||||
}
|
||||
active.settled = true;
|
||||
refs.activePlayback = null;
|
||||
logVoiceNative("playback_complete", {
|
||||
durationMs: Math.round(durationSec * 1000),
|
||||
});
|
||||
resolve(durationSec);
|
||||
}, durationSec * 1000);
|
||||
} catch (error) {
|
||||
clearPlaybackTimeout();
|
||||
const active = refs.activePlayback;
|
||||
if (active && !active.settled) {
|
||||
active.settled = true;
|
||||
refs.activePlayback = null;
|
||||
logVoiceNative("playback_failed", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
reject(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (refs.processingQueue || refs.queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
refs.processingQueue = true;
|
||||
logVoiceNative("queue_drain_start", {
|
||||
queueLength: refs.queue.length,
|
||||
});
|
||||
while (refs.queue.length > 0) {
|
||||
const item = refs.queue.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audio);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
logVoiceNative("queue_drain_complete");
|
||||
}
|
||||
|
||||
function stopLooping(): void {
|
||||
if (refs.looping.active) {
|
||||
logVoiceNative("thinking_tone_stop");
|
||||
}
|
||||
refs.looping.active = false;
|
||||
refs.looping.token += 1;
|
||||
if (refs.looping.timeout) {
|
||||
clearTimeout(refs.looping.timeout);
|
||||
refs.looping.timeout = null;
|
||||
}
|
||||
stopPlayback();
|
||||
}
|
||||
|
||||
return {
|
||||
async initialize() {
|
||||
await ensureInitialized();
|
||||
await ensureThinkingTone();
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
if (refs.destroyed) {
|
||||
logVoiceNative("engine_destroy_skipped", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
});
|
||||
return;
|
||||
}
|
||||
refs.destroyed = true;
|
||||
logVoiceNative("engine_destroy_start", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
});
|
||||
stopLooping();
|
||||
this.stop();
|
||||
this.clearQueue();
|
||||
if (refs.captureActive) {
|
||||
toggleRecording(false);
|
||||
refs.captureActive = false;
|
||||
}
|
||||
clearPlaybackTimeout();
|
||||
refs.muted = false;
|
||||
resetCaptureDebug();
|
||||
resetVolumeDebug();
|
||||
callbacks.onVolumeLevel(0);
|
||||
if (refs.initialized) {
|
||||
tearDown();
|
||||
refs.initialized = false;
|
||||
}
|
||||
microphoneSubscription.remove();
|
||||
logVoiceNative("listener_removed", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
event: "onMicrophoneData",
|
||||
});
|
||||
volumeSubscription.remove();
|
||||
logVoiceNative("listener_removed", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
event: "onInputVolumeLevelData",
|
||||
});
|
||||
logVoiceNative("engine_destroy_complete", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
});
|
||||
},
|
||||
|
||||
async startCapture() {
|
||||
if (refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
logVoiceNative("capture_start_requested");
|
||||
logVoiceNative("capture_trace", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
});
|
||||
await ensureMicrophonePermission();
|
||||
await ensureInitialized();
|
||||
toggleRecording(true);
|
||||
refs.captureActive = true;
|
||||
resetCaptureDebug();
|
||||
resetVolumeDebug();
|
||||
logVoiceNative("capture_started");
|
||||
} catch (error) {
|
||||
const wrapped = error instanceof Error ? error : new Error(String(error));
|
||||
logVoiceNative("capture_start_failed", {
|
||||
error: wrapped.message,
|
||||
});
|
||||
callbacks.onError?.(wrapped);
|
||||
throw wrapped;
|
||||
}
|
||||
},
|
||||
|
||||
async stopCapture() {
|
||||
if (refs.captureActive) {
|
||||
toggleRecording(false);
|
||||
}
|
||||
refs.captureActive = false;
|
||||
refs.muted = false;
|
||||
resetCaptureDebug();
|
||||
resetVolumeDebug();
|
||||
logVoiceNative("capture_stopped");
|
||||
logVoiceNative("capture_trace", {
|
||||
engineId,
|
||||
traceLabel,
|
||||
});
|
||||
callbacks.onVolumeLevel(0);
|
||||
},
|
||||
|
||||
toggleMute() {
|
||||
refs.muted = !refs.muted;
|
||||
logVoiceNative("mute_toggled", {
|
||||
muted: refs.muted,
|
||||
});
|
||||
if (refs.muted) {
|
||||
callbacks.onVolumeLevel(0);
|
||||
}
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
isMuted() {
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
async play(audio: AudioPlaybackSource) {
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
refs.queue.push({ audio, resolve, reject });
|
||||
logVoiceNative("queue_enqueue", {
|
||||
queueLength: refs.queue.length,
|
||||
blobBytes: audio.size,
|
||||
mimeType: audio.type || null,
|
||||
});
|
||||
if (!refs.processingQueue) {
|
||||
void processQueue();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
stop() {
|
||||
if (refs.activePlayback) {
|
||||
logVoiceNative("playback_stopped");
|
||||
}
|
||||
stopPlayback();
|
||||
clearPlaybackTimeout();
|
||||
const active = refs.activePlayback;
|
||||
refs.activePlayback = null;
|
||||
if (active && !active.settled) {
|
||||
active.settled = true;
|
||||
active.reject(new Error("Playback stopped"));
|
||||
}
|
||||
},
|
||||
|
||||
clearQueue() {
|
||||
if (refs.queue.length > 0) {
|
||||
logVoiceNative("queue_cleared", {
|
||||
queueLength: refs.queue.length,
|
||||
});
|
||||
}
|
||||
while (refs.queue.length > 0) {
|
||||
refs.queue.shift()!.reject(new Error("Playback stopped"));
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
},
|
||||
|
||||
isPlaying() {
|
||||
return refs.activePlayback !== null;
|
||||
},
|
||||
|
||||
playLooping(audio, gapMs) {
|
||||
if (refs.looping.active) {
|
||||
return;
|
||||
}
|
||||
|
||||
refs.looping.active = true;
|
||||
const token = refs.looping.token + 1;
|
||||
refs.looping.token = token;
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
await ensureInitialized();
|
||||
const cue =
|
||||
audio.byteLength > 0
|
||||
? {
|
||||
pcm16k: audio,
|
||||
durationMs: (audio.byteLength / 2 / 16000) * 1000,
|
||||
}
|
||||
: await ensureThinkingTone();
|
||||
logVoiceNative("thinking_tone_start", {
|
||||
bytes: cue.pcm16k.byteLength,
|
||||
durationMs: Math.round(cue.durationMs),
|
||||
gapMs: gapMs || THINKING_TONE_REPEAT_GAP_MS,
|
||||
});
|
||||
|
||||
const loop = () => {
|
||||
if (!refs.looping.active || refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
resumePlayback();
|
||||
playPCMData(cue.pcm16k);
|
||||
logVoiceNative("thinking_tone_tick", {
|
||||
durationMs: Math.round(cue.durationMs),
|
||||
});
|
||||
refs.looping.timeout = setTimeout(
|
||||
loop,
|
||||
cue.durationMs + (gapMs || THINKING_TONE_REPEAT_GAP_MS)
|
||||
);
|
||||
};
|
||||
|
||||
loop();
|
||||
} catch (error) {
|
||||
callbacks.onError?.(
|
||||
error instanceof Error ? error : new Error(String(error))
|
||||
);
|
||||
}
|
||||
})();
|
||||
},
|
||||
|
||||
stopLooping,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,548 @@
|
||||
import { getTauri } from "@/utils/tauri";
|
||||
import {
|
||||
loadThinkingToneArrayBuffer,
|
||||
THINKING_TONE_REPEAT_GAP_MS,
|
||||
} from "@/utils/thinking-tone";
|
||||
import type {
|
||||
AudioEngine,
|
||||
AudioEngineCallbacks,
|
||||
AudioPlaybackSource,
|
||||
} from "@/voice/audio-engine-types";
|
||||
|
||||
interface QueuedAudio {
|
||||
audio: AudioPlaybackSource;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
function getAudioContextCtor(): (typeof AudioContext) | null {
|
||||
if (typeof window === "undefined") {
|
||||
return null;
|
||||
}
|
||||
const browserWindow = window as typeof window & {
|
||||
webkitAudioContext?: typeof AudioContext;
|
||||
};
|
||||
return browserWindow.AudioContext ?? browserWindow.webkitAudioContext ?? null;
|
||||
}
|
||||
|
||||
function floatToInt16(sample: number): number {
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
|
||||
}
|
||||
|
||||
function resampleToPcm16(
|
||||
input: Float32Array,
|
||||
inputRate: number,
|
||||
outputRate: number
|
||||
): Uint8Array {
|
||||
if (input.length === 0) {
|
||||
return new Uint8Array(0);
|
||||
}
|
||||
|
||||
const ratio = inputRate / outputRate;
|
||||
const outputLength = Math.max(1, Math.round(input.length / ratio));
|
||||
const output = new Int16Array(outputLength);
|
||||
for (let i = 0; i < outputLength; i += 1) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const i1 = Math.min(input.length - 1, i0 + 1);
|
||||
const frac = sourceIndex - i0;
|
||||
const sample = input[i0]! * (1 - frac) + input[i1]! * frac;
|
||||
output[i] = floatToInt16(sample);
|
||||
}
|
||||
|
||||
return new Uint8Array(output.buffer, output.byteOffset, output.byteLength);
|
||||
}
|
||||
|
||||
function parsePcmSampleRate(mimeType: string): number | null {
|
||||
const match = /rate=(\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
}
|
||||
|
||||
function pcm16LeToAudioBuffer(
|
||||
context: AudioContext,
|
||||
bytes: Uint8Array,
|
||||
sampleRate: number
|
||||
): AudioBuffer {
|
||||
const sampleCount = Math.floor(bytes.length / 2);
|
||||
const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
|
||||
const channel = audioBuffer.getChannelData(0);
|
||||
for (let i = 0; i < sampleCount; i += 1) {
|
||||
const lo = bytes[i * 2]!;
|
||||
const hi = bytes[i * 2 + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value -= 0x10000;
|
||||
}
|
||||
channel[i] = value / 0x8000;
|
||||
}
|
||||
return audioBuffer;
|
||||
}
|
||||
|
||||
async function decodeAudioData(
|
||||
context: AudioContext,
|
||||
buffer: ArrayBuffer
|
||||
): Promise<AudioBuffer> {
|
||||
const maybePromise = context.decodeAudioData(buffer.slice(0));
|
||||
if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
|
||||
return maybePromise as Promise<AudioBuffer>;
|
||||
}
|
||||
return await new Promise<AudioBuffer>((resolve, reject) => {
|
||||
context.decodeAudioData(buffer.slice(0), resolve, reject);
|
||||
});
|
||||
}
|
||||
|
||||
export function createAudioEngine(
|
||||
callbacks: AudioEngineCallbacks,
|
||||
_options?: { traceLabel?: string }
|
||||
): AudioEngine {
|
||||
const refs: {
|
||||
playbackContext: AudioContext | null;
|
||||
captureContext: AudioContext | null;
|
||||
stream: MediaStream | null;
|
||||
source: MediaStreamAudioSourceNode | null;
|
||||
processor: ScriptProcessorNode | null;
|
||||
gain: GainNode | null;
|
||||
started: boolean;
|
||||
muted: boolean;
|
||||
queue: QueuedAudio[];
|
||||
processingQueue: boolean;
|
||||
activePlayback: {
|
||||
source: AudioBufferSourceNode;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
settled: boolean;
|
||||
} | null;
|
||||
looping: {
|
||||
token: number;
|
||||
source: AudioBufferSourceNode | null;
|
||||
gain: GainNode | null;
|
||||
timeout: ReturnType<typeof setTimeout> | null;
|
||||
};
|
||||
thinkingTone: {
|
||||
bytes: Uint8Array | null;
|
||||
buffer: AudioBuffer | null;
|
||||
};
|
||||
} = {
|
||||
playbackContext: null,
|
||||
captureContext: null,
|
||||
stream: null,
|
||||
source: null,
|
||||
processor: null,
|
||||
gain: null,
|
||||
started: false,
|
||||
muted: false,
|
||||
queue: [],
|
||||
processingQueue: false,
|
||||
activePlayback: null,
|
||||
looping: {
|
||||
token: 0,
|
||||
source: null,
|
||||
gain: null,
|
||||
timeout: null,
|
||||
},
|
||||
thinkingTone: {
|
||||
bytes: null,
|
||||
buffer: null,
|
||||
},
|
||||
};
|
||||
|
||||
async function ensurePlaybackContext(): Promise<AudioContext> {
|
||||
if (refs.playbackContext) {
|
||||
if (refs.playbackContext.state === "suspended") {
|
||||
await refs.playbackContext.resume().catch(() => undefined);
|
||||
}
|
||||
return refs.playbackContext;
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
if (context.state === "suspended") {
|
||||
await context.resume().catch(() => undefined);
|
||||
}
|
||||
refs.playbackContext = context;
|
||||
return context;
|
||||
}
|
||||
|
||||
async function ensureCaptureContext(): Promise<AudioContext> {
|
||||
if (refs.captureContext) {
|
||||
if (refs.captureContext.state === "suspended") {
|
||||
await refs.captureContext.resume().catch(() => undefined);
|
||||
}
|
||||
return refs.captureContext;
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
if (context.state === "suspended") {
|
||||
await context.resume().catch(() => undefined);
|
||||
}
|
||||
refs.captureContext = context;
|
||||
return context;
|
||||
}
|
||||
|
||||
async function ensureThinkingToneBytes(): Promise<Uint8Array> {
|
||||
if (refs.thinkingTone.bytes) {
|
||||
return refs.thinkingTone.bytes;
|
||||
}
|
||||
const wav = await loadThinkingToneArrayBuffer();
|
||||
refs.thinkingTone.bytes = new Uint8Array(wav);
|
||||
return refs.thinkingTone.bytes;
|
||||
}
|
||||
|
||||
async function ensureThinkingToneBuffer(): Promise<AudioBuffer> {
|
||||
if (refs.thinkingTone.buffer) {
|
||||
return refs.thinkingTone.buffer;
|
||||
}
|
||||
const context = await ensurePlaybackContext();
|
||||
const wav = await loadThinkingToneArrayBuffer();
|
||||
const audioBuffer = await decodeAudioData(context, wav);
|
||||
refs.thinkingTone.buffer = audioBuffer;
|
||||
return audioBuffer;
|
||||
}
|
||||
|
||||
function stopLooping(): void {
|
||||
refs.looping.token += 1;
|
||||
if (refs.looping.timeout) {
|
||||
clearTimeout(refs.looping.timeout);
|
||||
refs.looping.timeout = null;
|
||||
}
|
||||
if (refs.looping.source) {
|
||||
try {
|
||||
refs.looping.source.onended = null;
|
||||
refs.looping.source.stop();
|
||||
} catch {
|
||||
// Ignore best-effort stop errors.
|
||||
}
|
||||
refs.looping.source.disconnect();
|
||||
refs.looping.source = null;
|
||||
}
|
||||
if (refs.looping.gain) {
|
||||
refs.looping.gain.disconnect();
|
||||
refs.looping.gain = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
|
||||
const context = await ensurePlaybackContext();
|
||||
const arrayBuffer = await audio.arrayBuffer();
|
||||
const type = (audio.type || "").toLowerCase();
|
||||
const audioBuffer = type.startsWith("audio/pcm")
|
||||
? pcm16LeToAudioBuffer(
|
||||
context,
|
||||
new Uint8Array(arrayBuffer),
|
||||
parsePcmSampleRate(type) ?? 24000
|
||||
)
|
||||
: await decodeAudioData(context, arrayBuffer);
|
||||
|
||||
const durationSec = audioBuffer.duration;
|
||||
const source = context.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(context.destination);
|
||||
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
refs.activePlayback = { source, resolve, reject, settled: false };
|
||||
|
||||
const settle = (fn: () => void) => {
|
||||
const active = refs.activePlayback;
|
||||
if (!active || active.source !== source || active.settled) {
|
||||
return;
|
||||
}
|
||||
active.settled = true;
|
||||
refs.activePlayback = null;
|
||||
fn();
|
||||
};
|
||||
|
||||
source.onended = () => {
|
||||
settle(() => resolve(durationSec));
|
||||
};
|
||||
|
||||
try {
|
||||
source.start();
|
||||
} catch (error) {
|
||||
settle(() =>
|
||||
reject(error instanceof Error ? error : new Error(String(error)))
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (refs.processingQueue || refs.queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
refs.processingQueue = true;
|
||||
while (refs.queue.length > 0) {
|
||||
const item = refs.queue.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audio);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
}
|
||||
|
||||
async function stopCapture(): Promise<void> {
|
||||
refs.started = false;
|
||||
|
||||
try {
|
||||
refs.processor?.disconnect();
|
||||
refs.source?.disconnect();
|
||||
refs.gain?.disconnect();
|
||||
} catch {
|
||||
// Ignore best-effort teardown errors.
|
||||
}
|
||||
|
||||
if (refs.stream) {
|
||||
for (const track of refs.stream.getTracks()) {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
// Ignore best-effort teardown errors.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
refs.stream = null;
|
||||
refs.source = null;
|
||||
refs.processor = null;
|
||||
refs.gain = null;
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
|
||||
const captureContext = refs.captureContext;
|
||||
refs.captureContext = null;
|
||||
if (captureContext && captureContext.state !== "closed") {
|
||||
await captureContext.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async initialize() {
|
||||
await ensurePlaybackContext();
|
||||
await Promise.all([ensureThinkingToneBytes(), ensureThinkingToneBuffer()]);
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
stopLooping();
|
||||
this.stop();
|
||||
this.clearQueue();
|
||||
await stopCapture();
|
||||
|
||||
const playbackContext = refs.playbackContext;
|
||||
refs.playbackContext = null;
|
||||
refs.thinkingTone.buffer = null;
|
||||
if (playbackContext && playbackContext.state !== "closed") {
|
||||
await playbackContext.close().catch(() => undefined);
|
||||
}
|
||||
},
|
||||
|
||||
async startCapture() {
|
||||
if (refs.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const missingNavigator =
|
||||
typeof navigator === "undefined" ||
|
||||
!navigator.mediaDevices ||
|
||||
typeof navigator.mediaDevices.getUserMedia !== "function";
|
||||
const secureContext =
|
||||
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
|
||||
? window.isSecureContext
|
||||
: true;
|
||||
const currentOrigin =
|
||||
typeof window !== "undefined" && window.location
|
||||
? window.location.origin
|
||||
: "unknown";
|
||||
const isTauri = getTauri() !== null;
|
||||
|
||||
if (missingNavigator) {
|
||||
throw new Error("Microphone capture is not supported in this environment");
|
||||
}
|
||||
if (!secureContext && !isTauri) {
|
||||
throw new Error(
|
||||
`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const context = await ensureCaptureContext();
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const processor = context.createScriptProcessor(4096, 1, 1);
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 0;
|
||||
|
||||
processor.onaudioprocess = (event) => {
|
||||
if (!refs.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < input.length; i += 1) {
|
||||
const sample = input[i]!;
|
||||
sumSquares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||
const normalized = Math.min(1, Math.max(0, rms * 2));
|
||||
callbacks.onVolumeLevel(normalized);
|
||||
|
||||
if (refs.muted) {
|
||||
return;
|
||||
}
|
||||
|
||||
callbacks.onCaptureData(resampleToPcm16(input, context.sampleRate, 16000));
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(gain);
|
||||
gain.connect(context.destination);
|
||||
|
||||
refs.started = true;
|
||||
refs.stream = stream;
|
||||
refs.source = source;
|
||||
refs.processor = processor;
|
||||
refs.gain = gain;
|
||||
} catch (error) {
|
||||
await stopCapture();
|
||||
const wrapped = error instanceof Error ? error : new Error(String(error));
|
||||
callbacks.onError?.(wrapped);
|
||||
throw wrapped;
|
||||
}
|
||||
},
|
||||
|
||||
async stopCapture() {
|
||||
await stopCapture();
|
||||
},
|
||||
|
||||
toggleMute() {
|
||||
refs.muted = !refs.muted;
|
||||
if (refs.muted) {
|
||||
callbacks.onVolumeLevel(0);
|
||||
}
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
isMuted() {
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
async play(audio: AudioPlaybackSource) {
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
refs.queue.push({ audio, resolve, reject });
|
||||
if (!refs.processingQueue) {
|
||||
void processQueue();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
stop() {
|
||||
if (refs.activePlayback) {
|
||||
const active = refs.activePlayback;
|
||||
refs.activePlayback = null;
|
||||
try {
|
||||
active.source.onended = null;
|
||||
active.source.stop();
|
||||
} catch {
|
||||
// Ignore best-effort stop errors.
|
||||
}
|
||||
if (!active.settled) {
|
||||
active.settled = true;
|
||||
active.reject(new Error("Playback stopped"));
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
clearQueue() {
|
||||
while (refs.queue.length > 0) {
|
||||
refs.queue.shift()!.reject(new Error("Playback stopped"));
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
},
|
||||
|
||||
isPlaying() {
|
||||
return refs.activePlayback !== null;
|
||||
},
|
||||
|
||||
playLooping(audio, gapMs) {
|
||||
if (refs.looping.source || refs.looping.timeout) {
|
||||
return;
|
||||
}
|
||||
|
||||
const token = refs.looping.token + 1;
|
||||
refs.looping.token = token;
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const context = await ensurePlaybackContext();
|
||||
const audioBuffer =
|
||||
audio.byteLength > 0
|
||||
? pcm16LeToAudioBuffer(context, audio, 16000)
|
||||
: await ensureThinkingToneBuffer();
|
||||
if (refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 1;
|
||||
gain.connect(context.destination);
|
||||
refs.looping.gain = gain;
|
||||
|
||||
const playOnce = () => {
|
||||
if (refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
const source = context.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(gain);
|
||||
refs.looping.source = source;
|
||||
source.onended = () => {
|
||||
if (refs.looping.source === source) {
|
||||
refs.looping.source = null;
|
||||
}
|
||||
if (refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
refs.looping.timeout = setTimeout(
|
||||
playOnce,
|
||||
gapMs || THINKING_TONE_REPEAT_GAP_MS
|
||||
);
|
||||
};
|
||||
source.start();
|
||||
};
|
||||
|
||||
playOnce();
|
||||
} catch (error) {
|
||||
callbacks.onError?.(
|
||||
error instanceof Error ? error : new Error(String(error))
|
||||
);
|
||||
}
|
||||
})();
|
||||
},
|
||||
|
||||
stopLooping,
|
||||
};
|
||||
}
|
||||
@@ -139,6 +139,11 @@ export class SpeechSegmenter {
|
||||
}
|
||||
|
||||
flush(isLast: boolean): void {
|
||||
console.log("[SpeechSegmenter] flush", {
|
||||
isLast,
|
||||
audioBufferLength: this.audioBuffer.length,
|
||||
bufferedBytes: this.bufferedBytes,
|
||||
});
|
||||
if (this.audioBuffer.length === 0) {
|
||||
this.bufferedBytes = 0;
|
||||
return;
|
||||
@@ -157,6 +162,13 @@ export class SpeechSegmenter {
|
||||
* In non-continuous mode, chunk buffering is gated by the VAD state.
|
||||
*/
|
||||
pushPcmChunk(chunk: Uint8Array): void {
|
||||
console.log("[SpeechSegmenter] pushPcmChunk", {
|
||||
bytes: chunk.length,
|
||||
speechDetectionStartMs: this.speechDetectionStartMs,
|
||||
isSpeaking: this.isSpeaking,
|
||||
speechConfirmed: this.speechConfirmed,
|
||||
bufferedBytes: this.bufferedBytes,
|
||||
});
|
||||
if (chunk.length === 0) {
|
||||
return;
|
||||
}
|
||||
@@ -186,6 +198,16 @@ export class SpeechSegmenter {
|
||||
* Called with a normalized volume level (0..1). Drives VAD transitions.
|
||||
*/
|
||||
pushVolumeLevel(volume: number, nowMs: number): void {
|
||||
console.log("[SpeechSegmenter] pushVolumeLevel", {
|
||||
volume,
|
||||
nowMs,
|
||||
speechDetectionStartMs: this.speechDetectionStartMs,
|
||||
isSpeaking: this.isSpeaking,
|
||||
isDetecting: this.isDetecting,
|
||||
speechConfirmed: this.speechConfirmed,
|
||||
silenceStartMs: this.silenceStartMs,
|
||||
confirmedDropStartMs: this.confirmedDropStartMs,
|
||||
});
|
||||
if (this.config.enableContinuousStreaming) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { DaemonServerInfo } from "@/stores/session-store";
|
||||
import type { AudioEngine } from "@/voice/audio-engine-types";
|
||||
import {
|
||||
createVoiceRuntime,
|
||||
type VoiceRuntime,
|
||||
type VoiceSessionAdapter,
|
||||
} from "@/voice/voice-runtime";
|
||||
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
|
||||
|
||||
function createAudioEngineMock(): AudioEngine {
|
||||
return {
|
||||
initialize: vi.fn().mockResolvedValue(undefined),
|
||||
destroy: vi.fn().mockResolvedValue(undefined),
|
||||
startCapture: vi.fn().mockResolvedValue(undefined),
|
||||
stopCapture: vi.fn().mockResolvedValue(undefined),
|
||||
toggleMute: vi.fn().mockReturnValue(true),
|
||||
isMuted: vi.fn().mockReturnValue(false),
|
||||
play: vi.fn().mockResolvedValue(0.1),
|
||||
stop: vi.fn(),
|
||||
clearQueue: vi.fn(),
|
||||
isPlaying: vi.fn().mockReturnValue(false),
|
||||
playLooping: vi.fn(),
|
||||
stopLooping: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createSessionAdapter(serverId = "server-1"): VoiceSessionAdapter {
|
||||
return {
|
||||
serverId,
|
||||
setVoiceMode: vi.fn().mockResolvedValue(undefined),
|
||||
sendVoiceAudioChunk: vi.fn().mockResolvedValue(undefined),
|
||||
abortRequest: vi.fn().mockResolvedValue(undefined),
|
||||
setAssistantAudioPlaying: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createServerInfo(): DaemonServerInfo {
|
||||
return {
|
||||
serverId: "server-1",
|
||||
hostname: "host",
|
||||
version: "1.0.0",
|
||||
capabilities: {
|
||||
voice: {
|
||||
dictation: { enabled: true, reason: "" },
|
||||
voice: { enabled: true, reason: "" },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createRuntime(options?: {
|
||||
engine?: AudioEngine;
|
||||
getServerInfo?: (serverId: string) => DaemonServerInfo | null;
|
||||
}) {
|
||||
const engine = options?.engine ?? createAudioEngineMock();
|
||||
const runtime = createVoiceRuntime({
|
||||
engine,
|
||||
getServerInfo: options?.getServerInfo ?? (() => createServerInfo()),
|
||||
activateKeepAwake: vi.fn().mockResolvedValue(undefined),
|
||||
deactivateKeepAwake: vi.fn().mockResolvedValue(undefined),
|
||||
});
|
||||
|
||||
return { runtime, engine };
|
||||
}
|
||||
|
||||
describe("voice runtime", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("starts voice when adapter is ready", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
|
||||
expect(engine.initialize).toHaveBeenCalled();
|
||||
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
|
||||
expect(engine.startCapture).toHaveBeenCalled();
|
||||
expect(runtime.getSnapshot()).toMatchObject({
|
||||
phase: "listening",
|
||||
isVoiceMode: true,
|
||||
activeServerId: "server-1",
|
||||
activeAgentId: "agent-1",
|
||||
});
|
||||
});
|
||||
|
||||
it("transitions from capturing to submitting to waiting on the final chunk", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.handleCaptureVolume(0.4);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
|
||||
);
|
||||
runtime.handleCaptureVolume(0.4);
|
||||
runtime.handleCapturePcm(new Uint8Array(32000));
|
||||
expect(runtime.getSnapshot().phase).toBe("capturing");
|
||||
|
||||
runtime.handleCapturePcm(new Uint8Array(4000));
|
||||
runtime.handleCaptureVolume(0);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs + 10
|
||||
);
|
||||
runtime.handleCaptureVolume(0);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.silenceDurationMs + 10
|
||||
);
|
||||
runtime.handleCaptureVolume(0);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(adapter.sendVoiceAudioChunk).toHaveBeenLastCalledWith(
|
||||
expect.any(String),
|
||||
"audio/pcm;rate=16000;bits=16",
|
||||
true
|
||||
);
|
||||
expect(runtime.getSnapshot().phase).toBe("waiting");
|
||||
});
|
||||
|
||||
it("moves from waiting to playing on the first assistant audio", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
|
||||
expect(runtime.getSnapshot().phase).toBe("playing");
|
||||
expect(engine.stopLooping).toHaveBeenCalled();
|
||||
expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(true);
|
||||
});
|
||||
|
||||
it("returns to waiting after assistant playback when the turn is still active", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
runtime.onAssistantAudioFinished("server-1");
|
||||
|
||||
expect(runtime.getSnapshot().phase).toBe("waiting");
|
||||
expect(engine.playLooping).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns to listening after assistant playback once the turn is complete", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_completed");
|
||||
runtime.onAssistantAudioFinished("server-1");
|
||||
|
||||
expect(runtime.getSnapshot().phase).toBe("listening");
|
||||
expect(engine.playLooping).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("interrupts the active turn on barge-in after the grace period", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
|
||||
runtime.handleCaptureVolume(0.5);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs
|
||||
);
|
||||
|
||||
expect(adapter.abortRequest).toHaveBeenCalled();
|
||||
expect(engine.stop).toHaveBeenCalled();
|
||||
expect(runtime.getSnapshot().phase).toBe("capturing");
|
||||
});
|
||||
|
||||
it("authoritatively stops and suppresses later voice audio", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
await runtime.stopVoice();
|
||||
|
||||
expect(adapter.setVoiceMode).toHaveBeenLastCalledWith(false);
|
||||
expect(engine.stopCapture).toHaveBeenCalled();
|
||||
expect(runtime.getSnapshot().phase).toBe("disabled");
|
||||
expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns an explicit not-ready error when the adapter is missing", async () => {
|
||||
const { runtime } = createRuntime();
|
||||
await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow(
|
||||
"Voice runtime is not ready for host server-1"
|
||||
);
|
||||
});
|
||||
|
||||
it("resyncs voice mode after connection recovers", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
vi.mocked(adapter.setVoiceMode).mockClear();
|
||||
|
||||
runtime.updateSessionConnection("server-1", false);
|
||||
runtime.updateSessionConnection("server-1", true);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
|
||||
});
|
||||
|
||||
it("does not emit when the snapshot is unchanged", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
|
||||
const listener = vi.fn();
|
||||
const unsubscribe = runtime.subscribe(listener);
|
||||
|
||||
runtime.handleCaptureVolume(0);
|
||||
runtime.handleCaptureVolume(0);
|
||||
|
||||
expect(listener).not.toHaveBeenCalled();
|
||||
unsubscribe();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,859 @@
|
||||
import type { AgentStreamEventPayload } from "@server/shared/messages";
|
||||
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
|
||||
import type { DaemonServerInfo } from "@/stores/session-store";
|
||||
import type { AudioEngine } from "@/voice/audio-engine-types";
|
||||
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
|
||||
import { SpeechSegmenter } from "@/voice/speech-segmenter";
|
||||
|
||||
const PCM_MIME_TYPE = "audio/pcm;rate=16000;bits=16";
|
||||
const KEEP_AWAKE_TAG = "paseo:voice";
|
||||
const THINKING_TONE_REPEAT_GAP_MS = 350;
|
||||
const DISPLAY_VOLUME_PUBLISH_INTERVAL_MS = 120;
|
||||
const DISPLAY_VOLUME_CHANGE_EPSILON = 0.02;
|
||||
const DISPLAY_VOLUME_ATTACK = 0.35;
|
||||
const DISPLAY_VOLUME_RELEASE = 0.18;
|
||||
|
||||
type TurnEventType = Extract<
|
||||
AgentStreamEventPayload["type"],
|
||||
"turn_started" | "turn_completed" | "turn_failed" | "turn_canceled"
|
||||
>;
|
||||
|
||||
export type VoiceRuntimePhase =
|
||||
| "disabled"
|
||||
| "starting"
|
||||
| "listening"
|
||||
| "capturing"
|
||||
| "submitting"
|
||||
| "waiting"
|
||||
| "playing"
|
||||
| "stopping";
|
||||
|
||||
export interface VoiceRuntimeSnapshot {
|
||||
phase: VoiceRuntimePhase;
|
||||
isVoiceMode: boolean;
|
||||
isVoiceSwitching: boolean;
|
||||
isMuted: boolean;
|
||||
activeServerId: string | null;
|
||||
activeAgentId: string | null;
|
||||
}
|
||||
|
||||
export interface VoiceRuntimeTelemetrySnapshot {
|
||||
volume: number;
|
||||
isDetecting: boolean;
|
||||
isSpeaking: boolean;
|
||||
segmentDuration: number;
|
||||
}
|
||||
|
||||
export interface VoiceSessionAdapter {
|
||||
serverId: string;
|
||||
setVoiceMode(enabled: boolean, agentId?: string): Promise<void>;
|
||||
sendVoiceAudioChunk(
|
||||
audioData: string,
|
||||
mimeType: string,
|
||||
isLast: boolean
|
||||
): Promise<void>;
|
||||
abortRequest(): Promise<void>;
|
||||
setAssistantAudioPlaying(isPlaying: boolean): void;
|
||||
}
|
||||
|
||||
export interface VoiceRuntimeDeps {
|
||||
engine: AudioEngine;
|
||||
getServerInfo(serverId: string): DaemonServerInfo | null;
|
||||
activateKeepAwake(tag: string): Promise<void>;
|
||||
deactivateKeepAwake(tag: string): Promise<void>;
|
||||
}
|
||||
|
||||
interface RuntimeSessionState {
|
||||
adapter: VoiceSessionAdapter;
|
||||
connected: boolean;
|
||||
}
|
||||
|
||||
interface RuntimeState {
|
||||
snapshot: VoiceRuntimeSnapshot;
|
||||
telemetry: VoiceRuntimeTelemetrySnapshot;
|
||||
turnInProgress: boolean;
|
||||
transportReady: boolean;
|
||||
generation: number;
|
||||
speechInterruptTimer: ReturnType<typeof setTimeout> | null;
|
||||
speechInterruptSent: boolean;
|
||||
segmentDurationTimer: ReturnType<typeof setInterval> | null;
|
||||
lastDisplayVolumePublishMs: number;
|
||||
}
|
||||
|
||||
const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
|
||||
phase: "disabled",
|
||||
isVoiceMode: false,
|
||||
isVoiceSwitching: false,
|
||||
isMuted: false,
|
||||
activeServerId: null,
|
||||
activeAgentId: null,
|
||||
};
|
||||
|
||||
const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
|
||||
volume: 0,
|
||||
isDetecting: false,
|
||||
isSpeaking: false,
|
||||
segmentDuration: 0,
|
||||
};
|
||||
|
||||
function logVoiceRuntime(
|
||||
event: string,
|
||||
details?: Record<string, unknown>
|
||||
): void {
|
||||
if (details) {
|
||||
console.log(`[VoiceRuntime] ${event}`, details);
|
||||
return;
|
||||
}
|
||||
console.log(`[VoiceRuntime] ${event}`);
|
||||
}
|
||||
|
||||
function snapshotsEqual(
|
||||
left: VoiceRuntimeSnapshot,
|
||||
right: VoiceRuntimeSnapshot
|
||||
): boolean {
|
||||
return (
|
||||
left.phase === right.phase &&
|
||||
left.isVoiceMode === right.isVoiceMode &&
|
||||
left.isVoiceSwitching === right.isVoiceSwitching &&
|
||||
left.isMuted === right.isMuted &&
|
||||
left.activeServerId === right.activeServerId &&
|
||||
left.activeAgentId === right.activeAgentId
|
||||
);
|
||||
}
|
||||
|
||||
function telemetryEqual(
|
||||
left: VoiceRuntimeTelemetrySnapshot,
|
||||
right: VoiceRuntimeTelemetrySnapshot
|
||||
): boolean {
|
||||
return (
|
||||
left.volume === right.volume &&
|
||||
left.isDetecting === right.isDetecting &&
|
||||
left.isSpeaking === right.isSpeaking &&
|
||||
left.segmentDuration === right.segmentDuration
|
||||
);
|
||||
}
|
||||
|
||||
export interface VoiceRuntime {
|
||||
subscribe(listener: () => void): () => void;
|
||||
getSnapshot(): VoiceRuntimeSnapshot;
|
||||
subscribeTelemetry(listener: () => void): () => void;
|
||||
getTelemetrySnapshot(): VoiceRuntimeTelemetrySnapshot;
|
||||
registerSession(adapter: VoiceSessionAdapter): () => void;
|
||||
updateSessionConnection(serverId: string, connected: boolean): void;
|
||||
handleCapturePcm(chunk: Uint8Array): void;
|
||||
handleCaptureVolume(level: number): void;
|
||||
startVoice(serverId: string, agentId: string): Promise<void>;
|
||||
stopVoice(): Promise<void>;
|
||||
destroy(): Promise<void>;
|
||||
toggleMute(): void;
|
||||
isVoiceModeForAgent(serverId: string, agentId: string): boolean;
|
||||
shouldPlayVoiceAudio(serverId: string): boolean;
|
||||
onAssistantAudioStarted(serverId: string): void;
|
||||
onAssistantAudioFinished(serverId: string): void;
|
||||
onTranscriptionResult(serverId: string, text: string): void;
|
||||
onTurnEvent(serverId: string, agentId: string, eventType: TurnEventType): void;
|
||||
}
|
||||
|
||||
export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
const listeners = new Set<() => void>();
|
||||
const telemetryListeners = new Set<() => void>();
|
||||
const sessions = new Map<string, RuntimeSessionState>();
|
||||
const state: RuntimeState = {
|
||||
snapshot: INITIAL_SNAPSHOT,
|
||||
telemetry: INITIAL_TELEMETRY,
|
||||
turnInProgress: false,
|
||||
transportReady: false,
|
||||
generation: 0,
|
||||
speechInterruptTimer: null,
|
||||
speechInterruptSent: false,
|
||||
segmentDurationTimer: null,
|
||||
lastDisplayVolumePublishMs: 0,
|
||||
};
|
||||
|
||||
function emit(): void {
|
||||
for (const listener of listeners) {
|
||||
listener();
|
||||
}
|
||||
}
|
||||
|
||||
function emitTelemetry(): void {
|
||||
for (const listener of telemetryListeners) {
|
||||
listener();
|
||||
}
|
||||
}
|
||||
|
||||
function patchSnapshot(
|
||||
patch:
|
||||
| Partial<VoiceRuntimeSnapshot>
|
||||
| ((previous: VoiceRuntimeSnapshot) => VoiceRuntimeSnapshot)
|
||||
): void {
|
||||
const next =
|
||||
typeof patch === "function"
|
||||
? patch(state.snapshot)
|
||||
: { ...state.snapshot, ...patch };
|
||||
if (snapshotsEqual(next, state.snapshot)) {
|
||||
return;
|
||||
}
|
||||
const previous = state.snapshot;
|
||||
state.snapshot = next;
|
||||
logVoiceRuntime("snapshot_changed", {
|
||||
previousPhase: previous.phase,
|
||||
phase: next.phase,
|
||||
isVoiceMode: next.isVoiceMode,
|
||||
isVoiceSwitching: next.isVoiceSwitching,
|
||||
isMuted: next.isMuted,
|
||||
activeServerId: next.activeServerId,
|
||||
activeAgentId: next.activeAgentId,
|
||||
});
|
||||
emit();
|
||||
}
|
||||
|
||||
function patchTelemetry(
|
||||
patch:
|
||||
| Partial<VoiceRuntimeTelemetrySnapshot>
|
||||
| ((
|
||||
previous: VoiceRuntimeTelemetrySnapshot
|
||||
) => VoiceRuntimeTelemetrySnapshot)
|
||||
): void {
|
||||
const next =
|
||||
typeof patch === "function"
|
||||
? patch(state.telemetry)
|
||||
: { ...state.telemetry, ...patch };
|
||||
if (telemetryEqual(next, state.telemetry)) {
|
||||
return;
|
||||
}
|
||||
state.telemetry = next;
|
||||
emitTelemetry();
|
||||
}
|
||||
|
||||
function getActiveSession(): RuntimeSessionState | null {
|
||||
if (!state.snapshot.activeServerId) {
|
||||
return null;
|
||||
}
|
||||
return sessions.get(state.snapshot.activeServerId) ?? null;
|
||||
}
|
||||
|
||||
function clearSpeechInterruptTimer(): void {
|
||||
if (state.speechInterruptTimer) {
|
||||
clearTimeout(state.speechInterruptTimer);
|
||||
state.speechInterruptTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
function clearSegmentDurationTimer(): void {
|
||||
if (state.segmentDurationTimer) {
|
||||
clearInterval(state.segmentDurationTimer);
|
||||
state.segmentDurationTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
function reconcileSegmentDurationTimer(
|
||||
segmenter: SpeechSegmenter
|
||||
): void {
|
||||
if (!state.telemetry.isDetecting && !state.telemetry.isSpeaking) {
|
||||
clearSegmentDurationTimer();
|
||||
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.segmentDurationTimer) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.segmentDurationTimer = setInterval(() => {
|
||||
const startedAt = segmenter.getSpeechDetectionStartMs();
|
||||
patchTelemetry((prev) => ({
|
||||
...prev,
|
||||
segmentDuration: startedAt ? Date.now() - startedAt : 0,
|
||||
}));
|
||||
}, 100);
|
||||
}
|
||||
|
||||
function canPlayCue(): boolean {
|
||||
return (
|
||||
state.snapshot.isVoiceMode &&
|
||||
state.snapshot.phase === "waiting" &&
|
||||
!state.telemetry.isDetecting &&
|
||||
!state.telemetry.isSpeaking
|
||||
);
|
||||
}
|
||||
|
||||
function stopCue(): void {
|
||||
deps.engine.stopLooping();
|
||||
}
|
||||
|
||||
function resetSegmenter(segmenter: SpeechSegmenter): void {
|
||||
segmenter.reset();
|
||||
clearSpeechInterruptTimer();
|
||||
clearSegmentDurationTimer();
|
||||
patchTelemetry({ ...INITIAL_TELEMETRY });
|
||||
}
|
||||
|
||||
function reconcileCue(): void {
|
||||
if (!canPlayCue()) {
|
||||
stopCue();
|
||||
return;
|
||||
}
|
||||
logVoiceRuntime("thinking_tone_requested", {
|
||||
phase: state.snapshot.phase,
|
||||
isDetecting: state.telemetry.isDetecting,
|
||||
isSpeaking: state.telemetry.isSpeaking,
|
||||
});
|
||||
deps.engine.playLooping(new Uint8Array(0), THINKING_TONE_REPEAT_GAP_MS);
|
||||
}
|
||||
|
||||
async function interruptActiveTurn(): Promise<void> {
|
||||
const activeSession = getActiveSession();
|
||||
if (!activeSession) {
|
||||
return;
|
||||
}
|
||||
|
||||
stopCue();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
activeSession.adapter.setAssistantAudioPlaying(false);
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "capturing" }));
|
||||
|
||||
try {
|
||||
await activeSession.adapter.abortRequest();
|
||||
} catch (error) {
|
||||
console.error("[VoiceRuntime] Failed to abort active turn:", error);
|
||||
}
|
||||
}
|
||||
|
||||
const segmenter = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming: false,
|
||||
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
|
||||
confirmedDropGracePeriodMs:
|
||||
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
|
||||
silenceDurationMs: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
|
||||
speechConfirmationMs: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
|
||||
detectionGracePeriodMs: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
|
||||
},
|
||||
{
|
||||
onAudioSegment: ({ audioData, isLast }) => {
|
||||
const activeSession = getActiveSession();
|
||||
if (!activeSession || !state.transportReady || !state.snapshot.isVoiceMode) {
|
||||
return;
|
||||
}
|
||||
|
||||
const generation = state.generation;
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
phase: isLast ? "submitting" : "capturing",
|
||||
}));
|
||||
if (isLast) {
|
||||
state.turnInProgress = true;
|
||||
}
|
||||
logVoiceRuntime("audio_segment_ready", {
|
||||
isLast,
|
||||
base64Bytes: audioData.length,
|
||||
phase: state.snapshot.phase,
|
||||
serverId: activeSession.adapter.serverId,
|
||||
});
|
||||
|
||||
void activeSession.adapter
|
||||
.sendVoiceAudioChunk(audioData, PCM_MIME_TYPE, isLast)
|
||||
.then(() => {
|
||||
if (
|
||||
!isLast ||
|
||||
generation !== state.generation ||
|
||||
!state.snapshot.isVoiceMode ||
|
||||
state.snapshot.activeServerId !== activeSession.adapter.serverId ||
|
||||
state.snapshot.phase !== "submitting"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
logVoiceRuntime("audio_segment_sent", {
|
||||
isLast,
|
||||
serverId: activeSession.adapter.serverId,
|
||||
});
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
|
||||
reconcileCue();
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error("[VoiceRuntime] Failed to send audio segment:", error);
|
||||
});
|
||||
},
|
||||
onSpeechStart: () => {
|
||||
logVoiceRuntime("speech_started");
|
||||
stopCue();
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
logVoiceRuntime("speech_ended");
|
||||
clearSpeechInterruptTimer();
|
||||
state.speechInterruptSent = false;
|
||||
reconcileCue();
|
||||
},
|
||||
onDetectingChange: (isDetecting) => {
|
||||
const previous = state.telemetry;
|
||||
patchTelemetry((prev) => ({ ...prev, isDetecting }));
|
||||
if (previous.isDetecting !== isDetecting) {
|
||||
logVoiceRuntime("detecting_changed", {
|
||||
isDetecting,
|
||||
volume: Number(state.telemetry.volume.toFixed(3)),
|
||||
});
|
||||
}
|
||||
if (!previous.isDetecting && isDetecting) {
|
||||
stopCue();
|
||||
}
|
||||
reconcileSegmentDurationTimer(segmenter);
|
||||
reconcileCue();
|
||||
},
|
||||
onSpeakingChange: (isSpeaking) => {
|
||||
const previous = state.telemetry;
|
||||
patchTelemetry((prev) => ({ ...prev, isSpeaking }));
|
||||
if (previous.isSpeaking !== isSpeaking) {
|
||||
logVoiceRuntime("speaking_changed", {
|
||||
isSpeaking,
|
||||
phase: state.snapshot.phase,
|
||||
volume: Number(state.telemetry.volume.toFixed(3)),
|
||||
});
|
||||
}
|
||||
|
||||
if (!previous.isSpeaking && isSpeaking) {
|
||||
stopCue();
|
||||
if (
|
||||
state.snapshot.phase === "waiting" ||
|
||||
state.snapshot.phase === "playing"
|
||||
) {
|
||||
clearSpeechInterruptTimer();
|
||||
state.speechInterruptTimer = setTimeout(() => {
|
||||
state.speechInterruptTimer = null;
|
||||
if (
|
||||
state.speechInterruptSent ||
|
||||
!state.snapshot.isVoiceMode ||
|
||||
!state.telemetry.isSpeaking
|
||||
) {
|
||||
return;
|
||||
}
|
||||
state.speechInterruptSent = true;
|
||||
logVoiceRuntime("barge_in_interrupt_triggered", {
|
||||
phase: state.snapshot.phase,
|
||||
});
|
||||
void interruptActiveTurn();
|
||||
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
|
||||
logVoiceRuntime("barge_in_interrupt_scheduled", {
|
||||
gracePeriodMs: REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs,
|
||||
phase: state.snapshot.phase,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!isSpeaking) {
|
||||
clearSpeechInterruptTimer();
|
||||
state.speechInterruptSent = false;
|
||||
}
|
||||
|
||||
reconcileSegmentDurationTimer(segmenter);
|
||||
reconcileCue();
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
function resetToDisabledState(): void {
|
||||
state.transportReady = false;
|
||||
state.turnInProgress = false;
|
||||
state.speechInterruptSent = false;
|
||||
state.lastDisplayVolumePublishMs = 0;
|
||||
resetSegmenter(segmenter);
|
||||
patchSnapshot({ ...INITIAL_SNAPSHOT });
|
||||
}
|
||||
|
||||
function publishDisplayVolume(level: number, nowMs: number): void {
|
||||
const previousVolume = state.telemetry.volume;
|
||||
const smoothing =
|
||||
level >= previousVolume ? DISPLAY_VOLUME_ATTACK : DISPLAY_VOLUME_RELEASE;
|
||||
const nextVolume = Math.max(
|
||||
0,
|
||||
Math.min(1, previousVolume + (level - previousVolume) * smoothing)
|
||||
);
|
||||
const enoughTimeElapsed =
|
||||
nowMs - state.lastDisplayVolumePublishMs >=
|
||||
DISPLAY_VOLUME_PUBLISH_INTERVAL_MS;
|
||||
const enoughChange =
|
||||
Math.abs(nextVolume - previousVolume) >= DISPLAY_VOLUME_CHANGE_EPSILON;
|
||||
|
||||
if (!enoughTimeElapsed && !enoughChange) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.lastDisplayVolumePublishMs = nowMs;
|
||||
patchTelemetry((prev) => ({
|
||||
...prev,
|
||||
volume: Number(nextVolume.toFixed(3)),
|
||||
}));
|
||||
}
|
||||
|
||||
async function performLocalStop(): Promise<void> {
|
||||
stopCue();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
await deps.engine.stopCapture().catch(() => undefined);
|
||||
await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
getActiveSession()?.adapter.setAssistantAudioPlaying(false);
|
||||
resetToDisabledState();
|
||||
}
|
||||
|
||||
async function resyncVoiceMode(serverId: string): Promise<void> {
|
||||
if (
|
||||
!state.snapshot.isVoiceMode ||
|
||||
state.snapshot.activeServerId !== serverId ||
|
||||
!state.snapshot.activeAgentId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeSession = getActiveSession();
|
||||
if (!activeSession || !activeSession.connected) {
|
||||
return;
|
||||
}
|
||||
|
||||
patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: true }));
|
||||
try {
|
||||
await activeSession.adapter.setVoiceMode(true, state.snapshot.activeAgentId);
|
||||
state.transportReady = true;
|
||||
} finally {
|
||||
patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: false }));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
subscribe(listener) {
|
||||
listeners.add(listener);
|
||||
return () => {
|
||||
listeners.delete(listener);
|
||||
};
|
||||
},
|
||||
|
||||
getSnapshot() {
|
||||
return state.snapshot;
|
||||
},
|
||||
|
||||
subscribeTelemetry(listener) {
|
||||
telemetryListeners.add(listener);
|
||||
return () => {
|
||||
telemetryListeners.delete(listener);
|
||||
};
|
||||
},
|
||||
|
||||
getTelemetrySnapshot() {
|
||||
return state.telemetry;
|
||||
},
|
||||
|
||||
registerSession(adapter) {
|
||||
logVoiceRuntime("session_registered", {
|
||||
serverId: adapter.serverId,
|
||||
});
|
||||
sessions.set(adapter.serverId, {
|
||||
adapter,
|
||||
connected: true,
|
||||
});
|
||||
|
||||
return () => {
|
||||
const activeServerId = state.snapshot.activeServerId;
|
||||
sessions.delete(adapter.serverId);
|
||||
if (activeServerId === adapter.serverId) {
|
||||
void performLocalStop();
|
||||
}
|
||||
};
|
||||
},
|
||||
|
||||
updateSessionConnection(serverId, connected) {
|
||||
const session = sessions.get(serverId);
|
||||
if (!session) {
|
||||
return;
|
||||
}
|
||||
session.connected = connected;
|
||||
logVoiceRuntime("session_connection_changed", {
|
||||
serverId,
|
||||
connected,
|
||||
});
|
||||
if (state.snapshot.activeServerId !== serverId) {
|
||||
return;
|
||||
}
|
||||
if (!connected) {
|
||||
state.transportReady = false;
|
||||
return;
|
||||
}
|
||||
void resyncVoiceMode(serverId);
|
||||
},
|
||||
|
||||
handleCapturePcm(chunk) {
|
||||
console.log("[VoiceRuntime] handleCapturePcm", {
|
||||
bytes: chunk.byteLength,
|
||||
isVoiceMode: state.snapshot.isVoiceMode,
|
||||
isMuted: state.snapshot.isMuted,
|
||||
phase: state.snapshot.phase,
|
||||
});
|
||||
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
|
||||
return;
|
||||
}
|
||||
segmenter.pushPcmChunk(chunk);
|
||||
},
|
||||
|
||||
handleCaptureVolume(level) {
|
||||
const nowMs = Date.now();
|
||||
console.log("[VoiceRuntime] handleCaptureVolume", {
|
||||
level,
|
||||
isVoiceMode: state.snapshot.isVoiceMode,
|
||||
isMuted: state.snapshot.isMuted,
|
||||
phase: state.snapshot.phase,
|
||||
});
|
||||
const displayLevel = state.snapshot.isMuted ? 0 : level;
|
||||
publishDisplayVolume(displayLevel, nowMs);
|
||||
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
|
||||
return;
|
||||
}
|
||||
segmenter.pushVolumeLevel(level, nowMs);
|
||||
},
|
||||
|
||||
async startVoice(serverId, agentId) {
|
||||
logVoiceRuntime("start_requested", {
|
||||
serverId,
|
||||
agentId,
|
||||
});
|
||||
const session = sessions.get(serverId);
|
||||
if (!session) {
|
||||
throw new Error(`Voice runtime is not ready for host ${serverId}`);
|
||||
}
|
||||
if (!session.connected) {
|
||||
throw new Error(`Host ${serverId} is not connected`);
|
||||
}
|
||||
|
||||
const serverInfo = deps.getServerInfo(serverId);
|
||||
const unavailableMessage = resolveVoiceUnavailableMessage({
|
||||
serverInfo,
|
||||
mode: "voice",
|
||||
});
|
||||
if (unavailableMessage) {
|
||||
throw new Error(unavailableMessage);
|
||||
}
|
||||
|
||||
const previousServerId = state.snapshot.activeServerId;
|
||||
const previousAgentId = state.snapshot.activeAgentId;
|
||||
const generation = state.generation + 1;
|
||||
state.generation = generation;
|
||||
state.transportReady = false;
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isVoiceSwitching: true,
|
||||
phase: "starting",
|
||||
activeServerId: serverId,
|
||||
activeAgentId: agentId,
|
||||
}));
|
||||
|
||||
try {
|
||||
if (
|
||||
state.snapshot.isVoiceMode &&
|
||||
previousServerId &&
|
||||
(previousServerId !== serverId || previousAgentId !== agentId)
|
||||
) {
|
||||
const previousSession = sessions.get(previousServerId);
|
||||
if (previousSession) {
|
||||
previousSession.adapter.setAssistantAudioPlaying(false);
|
||||
await previousSession.adapter.setVoiceMode(false);
|
||||
}
|
||||
}
|
||||
|
||||
await deps.activateKeepAwake(KEEP_AWAKE_TAG).catch((error) => {
|
||||
console.warn("[VoiceRuntime] Failed to activate keep-awake:", error);
|
||||
});
|
||||
|
||||
await deps.engine.initialize();
|
||||
await session.adapter.setVoiceMode(true, agentId);
|
||||
await deps.engine.startCapture();
|
||||
if (state.generation !== generation) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.transportReady = true;
|
||||
state.turnInProgress = false;
|
||||
resetSegmenter(segmenter);
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isVoiceMode: true,
|
||||
isVoiceSwitching: false,
|
||||
phase: "listening",
|
||||
isMuted: deps.engine.isMuted(),
|
||||
}));
|
||||
logVoiceRuntime("start_completed", {
|
||||
serverId,
|
||||
agentId,
|
||||
});
|
||||
} catch (error) {
|
||||
logVoiceRuntime("start_failed", {
|
||||
serverId,
|
||||
agentId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
await performLocalStop();
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
|
||||
async stopVoice() {
|
||||
logVoiceRuntime("stop_requested", {
|
||||
activeServerId: state.snapshot.activeServerId,
|
||||
activeAgentId: state.snapshot.activeAgentId,
|
||||
});
|
||||
const activeSession = getActiveSession();
|
||||
const generation = state.generation + 1;
|
||||
state.generation = generation;
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isVoiceSwitching: true,
|
||||
phase: "stopping",
|
||||
}));
|
||||
|
||||
try {
|
||||
state.transportReady = false;
|
||||
stopCue();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
activeSession?.adapter.setAssistantAudioPlaying(false);
|
||||
if (activeSession) {
|
||||
await activeSession.adapter.setVoiceMode(false);
|
||||
}
|
||||
await deps.engine.stopCapture();
|
||||
await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
} finally {
|
||||
if (state.generation === generation) {
|
||||
resetToDisabledState();
|
||||
}
|
||||
logVoiceRuntime("stop_completed");
|
||||
}
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
await this.stopVoice().catch(() => undefined);
|
||||
await deps.engine.destroy();
|
||||
listeners.clear();
|
||||
telemetryListeners.clear();
|
||||
sessions.clear();
|
||||
},
|
||||
|
||||
toggleMute() {
|
||||
const nextMuted = deps.engine.toggleMute();
|
||||
logVoiceRuntime("mute_toggled", {
|
||||
muted: nextMuted,
|
||||
});
|
||||
if (nextMuted) {
|
||||
resetSegmenter(segmenter);
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isMuted: true,
|
||||
}));
|
||||
reconcileCue();
|
||||
return;
|
||||
}
|
||||
|
||||
patchSnapshot((prev) => ({ ...prev, isMuted: false }));
|
||||
},
|
||||
|
||||
isVoiceModeForAgent(serverId, agentId) {
|
||||
return (
|
||||
state.snapshot.isVoiceMode &&
|
||||
state.snapshot.activeServerId === serverId &&
|
||||
state.snapshot.activeAgentId === agentId
|
||||
);
|
||||
},
|
||||
|
||||
shouldPlayVoiceAudio(serverId) {
|
||||
return (
|
||||
state.snapshot.isVoiceMode &&
|
||||
state.snapshot.activeServerId === serverId &&
|
||||
state.snapshot.phase !== "stopping" &&
|
||||
state.snapshot.phase !== "disabled" &&
|
||||
!state.telemetry.isDetecting &&
|
||||
!state.telemetry.isSpeaking
|
||||
);
|
||||
},
|
||||
|
||||
onAssistantAudioStarted(serverId) {
|
||||
if (
|
||||
!state.snapshot.isVoiceMode ||
|
||||
state.snapshot.activeServerId !== serverId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
logVoiceRuntime("assistant_audio_started", {
|
||||
serverId,
|
||||
});
|
||||
stopCue();
|
||||
getActiveSession()?.adapter.setAssistantAudioPlaying(true);
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "playing" }));
|
||||
},
|
||||
|
||||
onAssistantAudioFinished(serverId) {
|
||||
if (state.snapshot.activeServerId !== serverId) {
|
||||
return;
|
||||
}
|
||||
|
||||
logVoiceRuntime("assistant_audio_finished", {
|
||||
serverId,
|
||||
turnInProgress: state.turnInProgress,
|
||||
});
|
||||
getActiveSession()?.adapter.setAssistantAudioPlaying(false);
|
||||
if (!state.snapshot.isVoiceMode) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.turnInProgress) {
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
|
||||
reconcileCue();
|
||||
return;
|
||||
}
|
||||
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
|
||||
reconcileCue();
|
||||
},
|
||||
|
||||
onTranscriptionResult(serverId, text) {
|
||||
if (
|
||||
serverId !== state.snapshot.activeServerId ||
|
||||
!state.snapshot.isVoiceMode
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (text.trim()) {
|
||||
return;
|
||||
}
|
||||
|
||||
logVoiceRuntime("empty_transcription_result", {
|
||||
serverId,
|
||||
});
|
||||
state.turnInProgress = false;
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
|
||||
stopCue();
|
||||
},
|
||||
|
||||
onTurnEvent(serverId, agentId, eventType) {
|
||||
if (
|
||||
!state.snapshot.isVoiceMode ||
|
||||
state.snapshot.activeServerId !== serverId ||
|
||||
state.snapshot.activeAgentId !== agentId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
logVoiceRuntime("turn_event", {
|
||||
serverId,
|
||||
agentId,
|
||||
eventType,
|
||||
});
|
||||
|
||||
if (eventType === "turn_started") {
|
||||
state.turnInProgress = true;
|
||||
return;
|
||||
}
|
||||
|
||||
state.turnInProgress = false;
|
||||
if (state.snapshot.phase !== "playing") {
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
|
||||
}
|
||||
stopCue();
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user