feat(voice): implement turn-detection and real-time audio streaming
This commit is contained in:
Mohamed Boudra
@@ -1,19 +1,9 @@
|
||||
import {
|
||||
addExpoTwoWayAudioEventListener,
|
||||
getMicrophonePermissionsAsync,
|
||||
initialize,
|
||||
playPCMData,
|
||||
requestMicrophonePermissionsAsync,
|
||||
resumePlayback,
|
||||
stopPlayback,
|
||||
tearDown,
|
||||
toggleRecording,
|
||||
} from "@boudra/expo-two-way-audio";
|
||||
import { Audio } from "expo-av";
|
||||
import * as FileSystem from "expo-file-system";
|
||||
import {
|
||||
THINKING_TONE_REPEAT_GAP_MS,
|
||||
} from "@/utils/thinking-tone";
|
||||
import {
|
||||
THINKING_TONE_NATIVE_PCM_BASE64,
|
||||
THINKING_TONE_NATIVE_PCM_DURATION_MS,
|
||||
} from "@/utils/thinking-tone.native-pcm";
|
||||
import type {
|
||||
@@ -21,7 +11,9 @@ import type {
|
||||
AudioEngineCallbacks,
|
||||
AudioPlaybackSource,
|
||||
} from "@/voice/audio-engine-types";
|
||||
import { Buffer } from "buffer";
|
||||
|
||||
// Use expo-av for playback instead of expo-two-way-audio (temporary test)
|
||||
const USE_EXPO_AV_PLAYBACK = true;
|
||||
|
||||
interface QueuedAudio {
|
||||
audio: AudioPlaybackSource;
|
||||
@@ -38,53 +30,6 @@ interface AudioEngineTraceOptions {
|
||||
traceLabel?: string;
|
||||
}
|
||||
|
||||
function resamplePcm16(
|
||||
pcm: Uint8Array,
|
||||
fromRate: number,
|
||||
toRate: number
|
||||
): Uint8Array {
|
||||
if (fromRate === toRate) {
|
||||
return pcm;
|
||||
}
|
||||
|
||||
const inputSamples = Math.floor(pcm.length / 2);
|
||||
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
|
||||
const output = new Uint8Array(outputSamples * 2);
|
||||
const ratio = fromRate / toRate;
|
||||
|
||||
const readInt16 = (sampleIndex: number): number => {
|
||||
const offset = sampleIndex * 2;
|
||||
if (offset + 1 >= pcm.length) {
|
||||
return 0;
|
||||
}
|
||||
const lo = pcm[offset]!;
|
||||
const hi = pcm[offset + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value -= 0x10000;
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const writeInt16 = (sampleIndex: number, value: number): void => {
|
||||
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
|
||||
const offset = sampleIndex * 2;
|
||||
output[offset] = clamped & 0xff;
|
||||
output[offset + 1] = (clamped >> 8) & 0xff;
|
||||
};
|
||||
|
||||
for (let i = 0; i < outputSamples; i += 1) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const frac = sourceIndex - i0;
|
||||
const s0 = readInt16(i0);
|
||||
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
|
||||
writeInt16(i, s0 + (s1 - s0) * frac);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
function parsePcmSampleRate(mimeType: string): number | null {
|
||||
const match = /rate=(\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
@@ -117,6 +62,7 @@ export function createAudioEngine(
|
||||
};
|
||||
thinkingTone: CuePcm | null;
|
||||
destroyed: boolean;
|
||||
mockVolumeInterval: ReturnType<typeof setInterval> | null;
|
||||
} = {
|
||||
initialized: false,
|
||||
captureActive: false,
|
||||
@@ -132,41 +78,58 @@ export function createAudioEngine(
|
||||
},
|
||||
thinkingTone: null,
|
||||
destroyed: false,
|
||||
mockVolumeInterval: null,
|
||||
};
|
||||
|
||||
const microphoneSubscription = addExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
(event) => {
|
||||
if (!refs.captureActive || refs.muted) {
|
||||
return;
|
||||
}
|
||||
callbacks.onCaptureData(event.data);
|
||||
}
|
||||
);
|
||||
// Only subscribe to native events if not mocking
|
||||
let microphoneSubscription: { remove(): void };
|
||||
let volumeSubscription: { remove(): void };
|
||||
|
||||
const volumeSubscription = addExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
(event) => {
|
||||
if (!refs.captureActive) {
|
||||
return;
|
||||
if (MOCK_MODE) {
|
||||
microphoneSubscription = { remove() {} };
|
||||
volumeSubscription = { remove() {} };
|
||||
} else {
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
microphoneSubscription = native.addExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
(event: any) => {
|
||||
if (!refs.captureActive || refs.muted) {
|
||||
return;
|
||||
}
|
||||
callbacks.onCaptureData(event.data);
|
||||
}
|
||||
const level = refs.muted ? 0 : event.data;
|
||||
callbacks.onVolumeLevel(level);
|
||||
}
|
||||
);
|
||||
);
|
||||
volumeSubscription = native.addExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
(event: any) => {
|
||||
if (!refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
const level = refs.muted ? 0 : event.data;
|
||||
callbacks.onVolumeLevel(level);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async function ensureInitialized(): Promise<void> {
|
||||
if (refs.initialized) {
|
||||
return;
|
||||
}
|
||||
await initialize();
|
||||
if (!MOCK_MODE) {
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
await native.initialize();
|
||||
}
|
||||
refs.initialized = true;
|
||||
}
|
||||
|
||||
async function ensureMicrophonePermission(): Promise<void> {
|
||||
let permission = await getMicrophonePermissionsAsync().catch(() => null);
|
||||
if (MOCK_MODE) {
|
||||
return;
|
||||
}
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
let permission = await native.getMicrophonePermissionsAsync().catch(() => null);
|
||||
if (!permission?.granted) {
|
||||
permission = await requestMicrophonePermissionsAsync().catch(() => null);
|
||||
permission = await native.requestMicrophonePermissionsAsync().catch(() => null);
|
||||
}
|
||||
if (!permission?.granted) {
|
||||
throw new Error(
|
||||
@@ -179,9 +142,8 @@ export function createAudioEngine(
|
||||
if (refs.thinkingTone) {
|
||||
return refs.thinkingTone;
|
||||
}
|
||||
const pcm16k = Buffer.from(THINKING_TONE_NATIVE_PCM_BASE64, "base64");
|
||||
const durationMs = THINKING_TONE_NATIVE_PCM_DURATION_MS;
|
||||
refs.thinkingTone = { pcm16k, durationMs };
|
||||
refs.thinkingTone = { pcm16k: new Uint8Array(0), durationMs };
|
||||
return refs.thinkingTone;
|
||||
}
|
||||
|
||||
@@ -194,7 +156,6 @@ export function createAudioEngine(
|
||||
|
||||
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
|
||||
await ensureInitialized();
|
||||
resumePlayback();
|
||||
|
||||
return await new Promise<number>(async (resolve, reject) => {
|
||||
refs.activePlayback = { resolve, reject, settled: false };
|
||||
@@ -203,10 +164,16 @@ export function createAudioEngine(
|
||||
const arrayBuffer = await audio.arrayBuffer();
|
||||
const pcm = new Uint8Array(arrayBuffer);
|
||||
const inputRate = parsePcmSampleRate(audio.type || "") ?? 24000;
|
||||
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
|
||||
const durationSec = pcm16k.length / 2 / 16000;
|
||||
const durationSec = pcm.length / 2 / inputRate;
|
||||
|
||||
console.log(`[AudioEngine.native] playPCMData: MOCK_MODE=${MOCK_MODE} inputRate=${inputRate} inputBytes=${pcm.length} durationSec=${durationSec.toFixed(3)}`);
|
||||
|
||||
if (!MOCK_MODE) {
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
native.resumePlayback();
|
||||
native.playPCMData(pcm);
|
||||
}
|
||||
|
||||
playPCMData(pcm16k);
|
||||
clearPlaybackTimeout();
|
||||
refs.playbackTimeout = setTimeout(() => {
|
||||
clearPlaybackTimeout();
|
||||
@@ -248,6 +215,13 @@ export function createAudioEngine(
|
||||
refs.processingQueue = false;
|
||||
}
|
||||
|
||||
function nativeStopPlayback(): void {
|
||||
if (!MOCK_MODE) {
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
native.stopPlayback();
|
||||
}
|
||||
}
|
||||
|
||||
function stopLooping(): void {
|
||||
refs.looping.active = false;
|
||||
refs.looping.token += 1;
|
||||
@@ -255,7 +229,7 @@ export function createAudioEngine(
|
||||
clearTimeout(refs.looping.timeout);
|
||||
refs.looping.timeout = null;
|
||||
}
|
||||
stopPlayback();
|
||||
nativeStopPlayback();
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -273,14 +247,22 @@ export function createAudioEngine(
|
||||
this.stop();
|
||||
this.clearQueue();
|
||||
if (refs.captureActive) {
|
||||
toggleRecording(false);
|
||||
if (!MOCK_MODE) {
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
native.toggleRecording(false);
|
||||
}
|
||||
if (refs.mockVolumeInterval) {
|
||||
clearInterval(refs.mockVolumeInterval);
|
||||
refs.mockVolumeInterval = null;
|
||||
}
|
||||
refs.captureActive = false;
|
||||
}
|
||||
clearPlaybackTimeout();
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
if (refs.initialized) {
|
||||
tearDown();
|
||||
if (refs.initialized && !MOCK_MODE) {
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
native.tearDown();
|
||||
refs.initialized = false;
|
||||
}
|
||||
microphoneSubscription.remove();
|
||||
@@ -295,7 +277,22 @@ export function createAudioEngine(
|
||||
try {
|
||||
await ensureMicrophonePermission();
|
||||
await ensureInitialized();
|
||||
toggleRecording(true);
|
||||
|
||||
if (MOCK_MODE) {
|
||||
// Emit fake volume levels and empty PCM data periodically
|
||||
refs.mockVolumeInterval = setInterval(() => {
|
||||
if (!refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
const fakeVolume = refs.muted ? 0 : 0.15 + Math.random() * 0.5;
|
||||
callbacks.onVolumeLevel(fakeVolume);
|
||||
// Send empty PCM chunk to keep the pipeline alive
|
||||
callbacks.onCaptureData(new Uint8Array(3200));
|
||||
}, 100);
|
||||
} else {
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
native.toggleRecording(true);
|
||||
}
|
||||
refs.captureActive = true;
|
||||
} catch (error) {
|
||||
const wrapped = error instanceof Error ? error : new Error(String(error));
|
||||
@@ -306,7 +303,15 @@ export function createAudioEngine(
|
||||
|
||||
async stopCapture() {
|
||||
if (refs.captureActive) {
|
||||
toggleRecording(false);
|
||||
if (MOCK_MODE) {
|
||||
if (refs.mockVolumeInterval) {
|
||||
clearInterval(refs.mockVolumeInterval);
|
||||
refs.mockVolumeInterval = null;
|
||||
}
|
||||
} else {
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
native.toggleRecording(false);
|
||||
}
|
||||
}
|
||||
refs.captureActive = false;
|
||||
refs.muted = false;
|
||||
@@ -335,7 +340,7 @@ export function createAudioEngine(
|
||||
},
|
||||
|
||||
stop() {
|
||||
stopPlayback();
|
||||
nativeStopPlayback();
|
||||
clearPlaybackTimeout();
|
||||
const active = refs.activePlayback;
|
||||
refs.activePlayback = null;
|
||||
@@ -368,20 +373,17 @@ export function createAudioEngine(
|
||||
void (async () => {
|
||||
try {
|
||||
await ensureInitialized();
|
||||
const cue =
|
||||
audio.byteLength > 0
|
||||
? {
|
||||
pcm16k: audio,
|
||||
durationMs: (audio.byteLength / 2 / 16000) * 1000,
|
||||
}
|
||||
: await ensureThinkingTone();
|
||||
const cue = await ensureThinkingTone();
|
||||
|
||||
const loop = () => {
|
||||
if (!refs.looping.active || refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
resumePlayback();
|
||||
playPCMData(cue.pcm16k);
|
||||
if (!MOCK_MODE) {
|
||||
const native = require("@boudra/expo-two-way-audio");
|
||||
native.resumePlayback();
|
||||
native.playPCMData(cue.pcm16k);
|
||||
}
|
||||
refs.looping.timeout = setTimeout(
|
||||
loop,
|
||||
cue.durationMs + (gapMs || THINKING_TONE_REPEAT_GAP_MS)
|
||||
|
||||
@@ -1,272 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { SpeechSegmenter } from "./speech-segmenter";
|
||||
|
||||
const mkPcmBytes = (length: number): Uint8Array => {
|
||||
const bytes = new Uint8Array(length);
|
||||
for (let i = 0; i < length; i++) bytes[i] = i % 255;
|
||||
return bytes;
|
||||
};
|
||||
|
||||
describe("SpeechSegmenter", () => {
|
||||
it("streams continuously and flushes at stop", () => {
|
||||
const onAudioSegment = vi.fn();
|
||||
const segmenter = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming: true,
|
||||
volumeThreshold: 0.3,
|
||||
silenceDurationMs: 2000,
|
||||
speechConfirmationMs: 300,
|
||||
detectionGracePeriodMs: 200,
|
||||
minChunkDurationMs: 100, // ensure we only flush on stop
|
||||
pcmSampleRate: 1000,
|
||||
},
|
||||
{ onAudioSegment }
|
||||
);
|
||||
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
expect(onAudioSegment).not.toHaveBeenCalled();
|
||||
|
||||
segmenter.stop(Date.now());
|
||||
|
||||
expect(onAudioSegment).toHaveBeenCalledTimes(1);
|
||||
const lastCall = onAudioSegment.mock.calls.at(-1)![0];
|
||||
expect(lastCall.isLast).toBe(true);
|
||||
expect(lastCall.audioData.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("detects speech, calls onSpeechStart, and flushes on speech end", () => {
|
||||
const onAudioSegment = vi.fn();
|
||||
const onSpeechStart = vi.fn();
|
||||
const onSpeechEnd = vi.fn();
|
||||
const detectingChanges: boolean[] = [];
|
||||
const speakingChanges: boolean[] = [];
|
||||
|
||||
const segmenter = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming: false,
|
||||
volumeThreshold: 0.3,
|
||||
silenceDurationMs: 2000,
|
||||
speechConfirmationMs: 300,
|
||||
detectionGracePeriodMs: 200,
|
||||
minChunkDurationMs: 100,
|
||||
pcmSampleRate: 1000,
|
||||
},
|
||||
{
|
||||
onAudioSegment,
|
||||
onSpeechStart,
|
||||
onSpeechEnd,
|
||||
onDetectingChange: (v) => detectingChanges.push(v),
|
||||
onSpeakingChange: (v) => speakingChanges.push(v),
|
||||
}
|
||||
);
|
||||
|
||||
const t0 = 10_000;
|
||||
|
||||
// Start detection.
|
||||
segmenter.pushVolumeLevel(0.5, t0);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
expect(detectingChanges).toEqual([true]);
|
||||
expect(speakingChanges).toEqual([true]);
|
||||
expect(onSpeechStart).not.toHaveBeenCalled();
|
||||
|
||||
// Confirm speech.
|
||||
segmenter.pushVolumeLevel(0.5, t0 + 300);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
expect(onSpeechStart).toHaveBeenCalledTimes(1);
|
||||
expect(detectingChanges).toEqual([true, false]);
|
||||
expect(speakingChanges).toEqual([true]);
|
||||
|
||||
// End speech after silence.
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 301);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000);
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000 + 2000);
|
||||
|
||||
expect(onSpeechEnd).toHaveBeenCalledTimes(1);
|
||||
expect(detectingChanges).toEqual([true, false, true, false]);
|
||||
expect(speakingChanges).toEqual([true, false]);
|
||||
expect(onAudioSegment).toHaveBeenCalled();
|
||||
const lastCall = onAudioSegment.mock.calls.at(-1)![0];
|
||||
expect(lastCall.isLast).toBe(true);
|
||||
expect(lastCall.audioData.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("keeps detection alive across a brief pause and confirms short utterances", () => {
|
||||
const onSpeechStart = vi.fn();
|
||||
const detectingChanges: boolean[] = [];
|
||||
|
||||
const segmenter = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming: false,
|
||||
volumeThreshold: 0.18,
|
||||
silenceDurationMs: 1400,
|
||||
speechConfirmationMs: 120,
|
||||
detectionGracePeriodMs: 700,
|
||||
minChunkDurationMs: 100,
|
||||
pcmSampleRate: 1000,
|
||||
},
|
||||
{
|
||||
onSpeechStart,
|
||||
onDetectingChange: (v) => detectingChanges.push(v),
|
||||
}
|
||||
);
|
||||
|
||||
const t0 = 20_000;
|
||||
|
||||
segmenter.pushVolumeLevel(0.4, t0);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(20));
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 80);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(20));
|
||||
segmenter.pushVolumeLevel(0.4, t0 + 140);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(20));
|
||||
|
||||
expect(detectingChanges).toContain(true);
|
||||
expect(onSpeechStart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps utterance open after fade-out and resumes speaking during silence grace", () => {
|
||||
const onAudioSegment = vi.fn();
|
||||
const onSpeechStart = vi.fn();
|
||||
const onSpeechEnd = vi.fn();
|
||||
const speakingChanges: boolean[] = [];
|
||||
|
||||
const segmenter = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming: false,
|
||||
volumeThreshold: 0.2,
|
||||
silenceDurationMs: 1500,
|
||||
speechConfirmationMs: 120,
|
||||
detectionGracePeriodMs: 400,
|
||||
minChunkDurationMs: 1000, // keep everything buffered until final flush
|
||||
pcmSampleRate: 1000,
|
||||
},
|
||||
{
|
||||
onAudioSegment,
|
||||
onSpeechStart,
|
||||
onSpeechEnd,
|
||||
onSpeakingChange: (v) => speakingChanges.push(v),
|
||||
}
|
||||
);
|
||||
|
||||
const t0 = 30_000;
|
||||
|
||||
// Start + confirm speaking.
|
||||
segmenter.pushVolumeLevel(0.6, t0);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(200));
|
||||
segmenter.pushVolumeLevel(0.6, t0 + 130);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(200));
|
||||
expect(onSpeechStart).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Drop below threshold: meter should fade, but turn remains open.
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 200);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(200));
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 500);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(200));
|
||||
|
||||
// Speak again before silence timeout expires: continue same utterance.
|
||||
segmenter.pushVolumeLevel(0.6, t0 + 900);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(200));
|
||||
expect(onSpeechStart).toHaveBeenCalledTimes(1);
|
||||
expect(onSpeechEnd).not.toHaveBeenCalled();
|
||||
|
||||
// End for real.
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 1000);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(200));
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 2600);
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 4200);
|
||||
|
||||
expect(speakingChanges).toEqual([true, false, true, false]);
|
||||
expect(onSpeechEnd).toHaveBeenCalledTimes(1);
|
||||
const finalCalls = onAudioSegment.mock.calls.filter((call) => call[0].isLast);
|
||||
expect(finalCalls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps confirmed speech active above release threshold even when below start threshold", () => {
|
||||
const onSpeechStart = vi.fn();
|
||||
const onSpeechEnd = vi.fn();
|
||||
const speakingChanges: boolean[] = [];
|
||||
|
||||
const segmenter = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming: false,
|
||||
volumeThreshold: 0.3,
|
||||
volumeReleaseThreshold: 0.1,
|
||||
silenceDurationMs: 1200,
|
||||
speechConfirmationMs: 120,
|
||||
detectionGracePeriodMs: 200,
|
||||
minChunkDurationMs: 1000,
|
||||
pcmSampleRate: 1000,
|
||||
},
|
||||
{
|
||||
onSpeechStart,
|
||||
onSpeechEnd,
|
||||
onSpeakingChange: (v) => speakingChanges.push(v),
|
||||
}
|
||||
);
|
||||
|
||||
const t0 = 40_000;
|
||||
|
||||
// Confirm speech.
|
||||
segmenter.pushVolumeLevel(0.4, t0);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
segmenter.pushVolumeLevel(0.4, t0 + 130);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
expect(onSpeechStart).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Stay between start and release thresholds for longer than silence timeout:
|
||||
// should still be considered speaking.
|
||||
segmenter.pushVolumeLevel(0.15, t0 + 400);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
segmenter.pushVolumeLevel(0.15, t0 + 1000);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
segmenter.pushVolumeLevel(0.15, t0 + 2000);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
|
||||
expect(onSpeechEnd).not.toHaveBeenCalled();
|
||||
expect(speakingChanges.at(-1)).toBe(true);
|
||||
|
||||
// Now actually drop below release threshold and allow silence timeout.
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 2001);
|
||||
segmenter.pushPcmChunk(mkPcmBytes(50));
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 3405);
|
||||
segmenter.pushVolumeLevel(0.0, t0 + 4706);
|
||||
|
||||
expect(onSpeechEnd).toHaveBeenCalledTimes(1);
|
||||
expect(speakingChanges.at(-1)).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores brief confirmed dips below release threshold without flip-flopping speaking", () => {
|
||||
const speakingChanges: boolean[] = [];
|
||||
|
||||
const segmenter = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming: false,
|
||||
volumeThreshold: 0.2,
|
||||
volumeReleaseThreshold: 0.08,
|
||||
confirmedDropGracePeriodMs: 250,
|
||||
silenceDurationMs: 1500,
|
||||
speechConfirmationMs: 120,
|
||||
detectionGracePeriodMs: 200,
|
||||
minChunkDurationMs: 1000,
|
||||
pcmSampleRate: 1000,
|
||||
},
|
||||
{
|
||||
onSpeakingChange: (v) => speakingChanges.push(v),
|
||||
}
|
||||
);
|
||||
|
||||
const t0 = 50_000;
|
||||
segmenter.pushVolumeLevel(0.4, t0);
|
||||
segmenter.pushVolumeLevel(0.4, t0 + 130); // confirm speaking
|
||||
|
||||
// Two short dips below release threshold; both shorter than debounce.
|
||||
segmenter.pushVolumeLevel(0.05, t0 + 300);
|
||||
segmenter.pushVolumeLevel(0.09, t0 + 500);
|
||||
segmenter.pushVolumeLevel(0.05, t0 + 620);
|
||||
segmenter.pushVolumeLevel(0.1, t0 + 760);
|
||||
|
||||
expect(speakingChanges).toEqual([true]);
|
||||
});
|
||||
});
|
||||
@@ -1,338 +0,0 @@
|
||||
import { Buffer } from "buffer";
|
||||
|
||||
export type SpeechSegment = { audioData: string; isLast: boolean };
|
||||
|
||||
export type SpeechSegmenterConfig = {
|
||||
enableContinuousStreaming: boolean;
|
||||
volumeThreshold: number; // 0..1
|
||||
volumeReleaseThreshold?: number; // 0..1; lower threshold used to keep active speech from dropping out
|
||||
confirmedDropGracePeriodMs?: number; // ms dip debounce before leaving confirmed speaking
|
||||
silenceDurationMs: number;
|
||||
speechConfirmationMs: number;
|
||||
detectionGracePeriodMs: number;
|
||||
minChunkDurationMs?: number;
|
||||
pcmSampleRate?: number;
|
||||
pcmChannels?: number;
|
||||
pcmBitsPerSample?: number;
|
||||
};
|
||||
|
||||
export type SpeechSegmenterCallbacks = {
|
||||
onAudioSegment?: (segment: SpeechSegment) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onDetectingChange?: (isDetecting: boolean) => void;
|
||||
onSpeakingChange?: (isSpeaking: boolean) => void;
|
||||
};
|
||||
|
||||
function uint8ArrayToBase64(bytes: Uint8Array): string {
|
||||
// NOTE: This is performance-sensitive during continuous streaming.
|
||||
// Buffer-backed base64 is significantly faster than manual string building.
|
||||
try {
|
||||
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
|
||||
} catch {
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
// eslint-disable-next-line no-restricted-globals
|
||||
return btoa(binary);
|
||||
}
|
||||
}
|
||||
|
||||
function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
|
||||
const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
|
||||
const result = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const arr of arrays) {
|
||||
result.set(arr, offset);
|
||||
offset += arr.length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export class SpeechSegmenter {
|
||||
private config: Required<Omit<SpeechSegmenterConfig, "minChunkDurationMs" | "pcmSampleRate" | "pcmChannels" | "pcmBitsPerSample">> &
|
||||
Required<Pick<SpeechSegmenterConfig, "minChunkDurationMs" | "pcmSampleRate" | "pcmChannels" | "pcmBitsPerSample">>;
|
||||
private callbacks: SpeechSegmenterCallbacks;
|
||||
|
||||
private audioBuffer: Uint8Array[] = [];
|
||||
private bufferedBytes = 0;
|
||||
|
||||
private speechDetectionStartMs: number | null = null;
|
||||
private detectionSilenceStartMs: number | null = null;
|
||||
private confirmedDropStartMs: number | null = null;
|
||||
private silenceStartMs: number | null = null;
|
||||
private isSpeaking = false;
|
||||
private isDetecting = false;
|
||||
private speechConfirmed = false;
|
||||
|
||||
private pcmBytesPerMs: number;
|
||||
private minChunkBytes: number;
|
||||
|
||||
constructor(config: SpeechSegmenterConfig, callbacks: SpeechSegmenterCallbacks = {}) {
|
||||
const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold(
|
||||
config.volumeThreshold,
|
||||
config.volumeReleaseThreshold
|
||||
);
|
||||
this.config = {
|
||||
...config,
|
||||
volumeReleaseThreshold,
|
||||
confirmedDropGracePeriodMs: config.confirmedDropGracePeriodMs ?? 250,
|
||||
minChunkDurationMs: config.minChunkDurationMs ?? 1000,
|
||||
pcmSampleRate: config.pcmSampleRate ?? 16000,
|
||||
pcmChannels: config.pcmChannels ?? 1,
|
||||
pcmBitsPerSample: config.pcmBitsPerSample ?? 16,
|
||||
};
|
||||
this.callbacks = callbacks;
|
||||
this.pcmBytesPerMs = 0;
|
||||
this.minChunkBytes = 0;
|
||||
this.recomputeDerived();
|
||||
}
|
||||
|
||||
updateConfig(next: SpeechSegmenterConfig): void {
|
||||
const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold(
|
||||
next.volumeThreshold,
|
||||
next.volumeReleaseThreshold
|
||||
);
|
||||
this.config = {
|
||||
...next,
|
||||
volumeReleaseThreshold,
|
||||
confirmedDropGracePeriodMs: next.confirmedDropGracePeriodMs ?? this.config.confirmedDropGracePeriodMs,
|
||||
minChunkDurationMs: next.minChunkDurationMs ?? this.config.minChunkDurationMs,
|
||||
pcmSampleRate: next.pcmSampleRate ?? this.config.pcmSampleRate,
|
||||
pcmChannels: next.pcmChannels ?? this.config.pcmChannels,
|
||||
pcmBitsPerSample: next.pcmBitsPerSample ?? this.config.pcmBitsPerSample,
|
||||
};
|
||||
this.recomputeDerived();
|
||||
}
|
||||
|
||||
setCallbacks(callbacks: SpeechSegmenterCallbacks): void {
|
||||
this.callbacks = callbacks;
|
||||
}
|
||||
|
||||
getSpeechDetectionStartMs(): number | null {
|
||||
return this.speechDetectionStartMs;
|
||||
}
|
||||
|
||||
getIsSpeaking(): boolean {
|
||||
return this.isSpeaking;
|
||||
}
|
||||
|
||||
getIsDetecting(): boolean {
|
||||
return this.isDetecting;
|
||||
}
|
||||
|
||||
getVolumeReleaseThreshold(): number {
|
||||
return this.config.volumeReleaseThreshold;
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.audioBuffer = [];
|
||||
this.bufferedBytes = 0;
|
||||
this.speechDetectionStartMs = null;
|
||||
this.detectionSilenceStartMs = null;
|
||||
this.confirmedDropStartMs = null;
|
||||
this.silenceStartMs = null;
|
||||
this.isSpeaking = false;
|
||||
this.isDetecting = false;
|
||||
this.speechConfirmed = false;
|
||||
}
|
||||
|
||||
flush(isLast: boolean): void {
|
||||
if (this.audioBuffer.length === 0) {
|
||||
this.bufferedBytes = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
const combinedBinary = concatenateUint8Arrays(this.audioBuffer);
|
||||
const pcmBase64 = uint8ArrayToBase64(combinedBinary);
|
||||
this.callbacks.onAudioSegment?.({ audioData: pcmBase64, isLast });
|
||||
|
||||
this.audioBuffer = [];
|
||||
this.bufferedBytes = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when the input stream emits a PCM16 chunk (Uint8Array of bytes).
|
||||
* In non-continuous mode, chunk buffering is gated by the VAD state.
|
||||
*/
|
||||
pushPcmChunk(chunk: Uint8Array): void {
|
||||
if (chunk.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.config.enableContinuousStreaming) {
|
||||
this.audioBuffer.push(chunk);
|
||||
this.bufferedBytes += chunk.length;
|
||||
if (this.bufferedBytes >= this.minChunkBytes) {
|
||||
this.flush(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Buffer if we're detecting or speaking. When speech is confirmed, we also
|
||||
// stream partial chunks to reduce latency.
|
||||
if (this.speechDetectionStartMs !== null || this.isSpeaking) {
|
||||
this.audioBuffer.push(chunk);
|
||||
this.bufferedBytes += chunk.length;
|
||||
|
||||
if (this.speechConfirmed && this.bufferedBytes >= this.minChunkBytes) {
|
||||
this.flush(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called with a normalized volume level (0..1). Drives VAD transitions.
|
||||
*/
|
||||
pushVolumeLevel(volume: number, nowMs: number): void {
|
||||
if (this.config.enableContinuousStreaming) {
|
||||
return;
|
||||
}
|
||||
|
||||
const speechDetectedForStart = volume > this.config.volumeThreshold;
|
||||
const speechDetectedForContinue = volume > this.config.volumeReleaseThreshold;
|
||||
|
||||
if (!this.speechConfirmed) {
|
||||
if (speechDetectedForStart) {
|
||||
// Initial detection phase.
|
||||
this.confirmedDropStartMs = null;
|
||||
if (this.speechDetectionStartMs === null) {
|
||||
this.speechDetectionStartMs = nowMs;
|
||||
this.detectionSilenceStartMs = null;
|
||||
this.audioBuffer = [];
|
||||
this.bufferedBytes = 0;
|
||||
this.setDetecting(true);
|
||||
this.setSpeaking(true);
|
||||
return;
|
||||
}
|
||||
|
||||
// Volume is back above threshold - reset grace period.
|
||||
this.detectionSilenceStartMs = null;
|
||||
this.setSpeaking(true);
|
||||
|
||||
const speechDuration = nowMs - this.speechDetectionStartMs;
|
||||
if (speechDuration >= this.config.speechConfirmationMs) {
|
||||
this.speechConfirmed = true;
|
||||
this.silenceStartMs = null;
|
||||
this.setDetecting(false);
|
||||
this.callbacks.onSpeechStart?.();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.speechDetectionStartMs !== null) {
|
||||
// Volume dropped during detection phase - apply grace period.
|
||||
if (this.detectionSilenceStartMs === null) {
|
||||
this.detectionSilenceStartMs = nowMs;
|
||||
this.setSpeaking(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const graceDuration = nowMs - this.detectionSilenceStartMs;
|
||||
if (graceDuration >= this.config.detectionGracePeriodMs) {
|
||||
// Cancel detection.
|
||||
this.speechDetectionStartMs = null;
|
||||
this.detectionSilenceStartMs = null;
|
||||
this.confirmedDropStartMs = null;
|
||||
this.audioBuffer = [];
|
||||
this.bufferedBytes = 0;
|
||||
this.setDetecting(false);
|
||||
this.setSpeaking(false);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (speechDetectedForContinue) {
|
||||
// Continuing confirmed speech (or resuming during silence grace).
|
||||
this.confirmedDropStartMs = null;
|
||||
this.silenceStartMs = null;
|
||||
this.setDetecting(false);
|
||||
this.setSpeaking(true);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.confirmedDropStartMs === null) {
|
||||
this.confirmedDropStartMs = nowMs;
|
||||
return;
|
||||
}
|
||||
|
||||
const confirmedDropDuration = nowMs - this.confirmedDropStartMs;
|
||||
if (confirmedDropDuration < this.config.confirmedDropGracePeriodMs) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Potential speech END while utterance remains open.
|
||||
if (this.silenceStartMs === null) {
|
||||
this.silenceStartMs = nowMs;
|
||||
this.setDetecting(true);
|
||||
this.setSpeaking(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const silenceDuration = nowMs - this.silenceStartMs;
|
||||
if (silenceDuration >= this.config.silenceDurationMs) {
|
||||
// Speech END confirmed.
|
||||
this.speechConfirmed = false;
|
||||
this.speechDetectionStartMs = null;
|
||||
this.detectionSilenceStartMs = null;
|
||||
this.confirmedDropStartMs = null;
|
||||
this.silenceStartMs = null;
|
||||
this.setDetecting(false);
|
||||
this.setSpeaking(false);
|
||||
this.callbacks.onSpeechEnd?.();
|
||||
if (this.audioBuffer.length === 0) {
|
||||
// Important: downstream expects an explicit segment boundary marker.
|
||||
this.callbacks.onAudioSegment?.({ audioData: "", isLast: true });
|
||||
} else {
|
||||
this.flush(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stop(nowMs: number): void {
|
||||
void nowMs;
|
||||
if (this.config.enableContinuousStreaming) {
|
||||
this.flush(true);
|
||||
}
|
||||
// Ensure observers see state clear transitions.
|
||||
this.setDetecting(false);
|
||||
this.setSpeaking(false);
|
||||
this.reset();
|
||||
}
|
||||
|
||||
private setDetecting(next: boolean): void {
|
||||
if (this.isDetecting === next) {
|
||||
return;
|
||||
}
|
||||
this.isDetecting = next;
|
||||
this.callbacks.onDetectingChange?.(next);
|
||||
}
|
||||
|
||||
private setSpeaking(next: boolean): void {
|
||||
if (this.isSpeaking === next) {
|
||||
return;
|
||||
}
|
||||
this.isSpeaking = next;
|
||||
this.callbacks.onSpeakingChange?.(next);
|
||||
}
|
||||
|
||||
private recomputeDerived(): void {
|
||||
this.pcmBytesPerMs =
|
||||
(this.config.pcmSampleRate *
|
||||
this.config.pcmChannels *
|
||||
(this.config.pcmBitsPerSample / 8)) /
|
||||
1000;
|
||||
this.minChunkBytes = Math.round(this.pcmBytesPerMs * this.config.minChunkDurationMs);
|
||||
}
|
||||
|
||||
private resolveVolumeReleaseThreshold(
|
||||
volumeThreshold: number,
|
||||
configured?: number
|
||||
): number {
|
||||
const defaultReleaseThreshold = volumeThreshold * 0.4;
|
||||
const raw = configured ?? defaultReleaseThreshold;
|
||||
return Math.max(0, Math.min(raw, volumeThreshold));
|
||||
}
|
||||
}
|
||||
@@ -30,11 +30,30 @@ function createSessionAdapter(serverId = "server-1"): VoiceSessionAdapter {
|
||||
serverId,
|
||||
setVoiceMode: vi.fn().mockResolvedValue(undefined),
|
||||
sendVoiceAudioChunk: vi.fn().mockResolvedValue(undefined),
|
||||
audioPlayed: vi.fn().mockResolvedValue(undefined),
|
||||
abortRequest: vi.fn().mockResolvedValue(undefined),
|
||||
setAssistantAudioPlaying: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createAudioPayload(args: {
|
||||
id: string;
|
||||
groupId: string;
|
||||
chunkIndex: number;
|
||||
isLastChunk: boolean;
|
||||
isVoiceMode?: boolean;
|
||||
}) {
|
||||
return {
|
||||
id: args.id,
|
||||
groupId: args.groupId,
|
||||
chunkIndex: args.chunkIndex,
|
||||
isLastChunk: args.isLastChunk,
|
||||
isVoiceMode: args.isVoiceMode ?? true,
|
||||
format: "pcm",
|
||||
audio: Buffer.from(`chunk-${args.chunkIndex}`).toString("base64"),
|
||||
} as const;
|
||||
}
|
||||
|
||||
function createServerInfo(): DaemonServerInfo {
|
||||
return {
|
||||
serverId: "server-1",
|
||||
@@ -91,37 +110,21 @@ describe("voice runtime", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("transitions from capturing to submitting to waiting on the final chunk", async () => {
|
||||
it("streams continuous PCM chunks and waits after receiving a transcript", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.handleCaptureVolume(0.4);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
|
||||
);
|
||||
runtime.handleCaptureVolume(0.4);
|
||||
runtime.handleCapturePcm(new Uint8Array(32000));
|
||||
expect(runtime.getSnapshot().phase).toBe("capturing");
|
||||
|
||||
runtime.handleCapturePcm(new Uint8Array(4000));
|
||||
runtime.handleCaptureVolume(0);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs + 10
|
||||
);
|
||||
runtime.handleCaptureVolume(0);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.silenceDurationMs + 10
|
||||
);
|
||||
runtime.handleCaptureVolume(0);
|
||||
await Promise.resolve();
|
||||
expect(runtime.getSnapshot().phase).toBe("listening");
|
||||
|
||||
expect(adapter.sendVoiceAudioChunk).toHaveBeenLastCalledWith(
|
||||
expect.any(String),
|
||||
"audio/pcm;rate=16000;bits=16",
|
||||
true
|
||||
"audio/pcm;rate=16000;bits=16"
|
||||
);
|
||||
|
||||
runtime.onTranscriptionResult("server-1", "hello");
|
||||
expect(runtime.getSnapshot().phase).toBe("waiting");
|
||||
});
|
||||
|
||||
@@ -138,6 +141,110 @@ describe("voice runtime", () => {
|
||||
expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(true);
|
||||
});
|
||||
|
||||
it("starts playback on the first chunk before isLastChunk arrives", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
let resolvePlay!: (duration: number) => void;
|
||||
const engine = createAudioEngineMock();
|
||||
vi.mocked(engine.play).mockImplementation(
|
||||
() =>
|
||||
new Promise<number>((resolve) => {
|
||||
resolvePlay = resolve;
|
||||
})
|
||||
);
|
||||
const { runtime } = createRuntime({ engine });
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
||||
|
||||
runtime.handleAudioOutput(
|
||||
"server-1",
|
||||
createAudioPayload({
|
||||
id: "chunk-0",
|
||||
groupId: "group-1",
|
||||
chunkIndex: 0,
|
||||
isLastChunk: false,
|
||||
})
|
||||
);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(engine.play).toHaveBeenCalledTimes(1);
|
||||
expect(runtime.getSnapshot().phase).toBe("playing");
|
||||
expect(adapter.audioPlayed).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
runtime.handleAudioOutput(
|
||||
"server-1",
|
||||
createAudioPayload({
|
||||
id: "chunk-1",
|
||||
groupId: "group-1",
|
||||
chunkIndex: 1,
|
||||
isLastChunk: true,
|
||||
})
|
||||
);
|
||||
|
||||
expect(engine.play).toHaveBeenCalledTimes(1);
|
||||
resolvePlay(0.1);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(adapter.audioPlayed).toHaveBeenCalledWith("chunk-0");
|
||||
expect(engine.play).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
it("acknowledges chunks only after they are consumed and finishes after the final chunk", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const playResolvers: Array<(duration: number) => void> = [];
|
||||
const engine = createAudioEngineMock();
|
||||
vi.mocked(engine.play).mockImplementation(
|
||||
() =>
|
||||
new Promise<number>((resolve) => {
|
||||
playResolvers.push(resolve);
|
||||
})
|
||||
);
|
||||
const { runtime } = createRuntime({ engine });
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
||||
|
||||
runtime.handleAudioOutput(
|
||||
"server-1",
|
||||
createAudioPayload({
|
||||
id: "chunk-0",
|
||||
groupId: "group-1",
|
||||
chunkIndex: 0,
|
||||
isLastChunk: false,
|
||||
})
|
||||
);
|
||||
runtime.handleAudioOutput(
|
||||
"server-1",
|
||||
createAudioPayload({
|
||||
id: "chunk-1",
|
||||
groupId: "group-1",
|
||||
chunkIndex: 1,
|
||||
isLastChunk: true,
|
||||
})
|
||||
);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(engine.play).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
expect(adapter.audioPlayed).not.toHaveBeenCalled();
|
||||
|
||||
playResolvers.shift()!(0.1);
|
||||
await vi.waitFor(() => {
|
||||
expect(adapter.audioPlayed).toHaveBeenCalledWith("chunk-0");
|
||||
expect(engine.play).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
playResolvers.shift()!(0.1);
|
||||
await vi.waitFor(() => {
|
||||
expect(adapter.audioPlayed).toHaveBeenCalledWith("chunk-1");
|
||||
expect(runtime.getSnapshot().phase).toBe("waiting");
|
||||
});
|
||||
});
|
||||
|
||||
it("returns to waiting after assistant playback when the turn is still active", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
@@ -152,6 +259,18 @@ describe("voice runtime", () => {
|
||||
expect(engine.playLooping).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("starts the thinking tone when an agent turn begins before playback", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
||||
|
||||
expect(runtime.getSnapshot().phase).toBe("waiting");
|
||||
expect(engine.playLooping).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns to listening after assistant playback once the turn is complete", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
@@ -164,10 +283,11 @@ describe("voice runtime", () => {
|
||||
runtime.onAssistantAudioFinished("server-1");
|
||||
|
||||
expect(runtime.getSnapshot().phase).toBe("listening");
|
||||
expect(engine.playLooping).not.toHaveBeenCalled();
|
||||
expect(engine.playLooping).toHaveBeenCalledTimes(1);
|
||||
expect(engine.stopLooping).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("interrupts the active turn on barge-in after the grace period", async () => {
|
||||
it("keeps local volume alone non-authoritative for playback interruption", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
@@ -177,16 +297,46 @@ describe("voice runtime", () => {
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
|
||||
runtime.handleCaptureVolume(0.5);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs
|
||||
);
|
||||
expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(false);
|
||||
expect(adapter.abortRequest).not.toHaveBeenCalled();
|
||||
expect(engine.stop).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps the meter white state driven by server speech detection", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.handleCaptureVolume(0.7);
|
||||
|
||||
expect(runtime.getTelemetrySnapshot()).toMatchObject({
|
||||
volume: expect.any(Number),
|
||||
isDetecting: true,
|
||||
isSpeaking: false,
|
||||
});
|
||||
|
||||
runtime.onServerSpeechStateChanged("server-1", true);
|
||||
expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(true);
|
||||
|
||||
runtime.onServerSpeechStateChanged("server-1", false);
|
||||
expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(false);
|
||||
});
|
||||
|
||||
it("stops assistant playback immediately when server speech starts", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
|
||||
runtime.onServerSpeechStateChanged("server-1", true);
|
||||
|
||||
expect(adapter.abortRequest).toHaveBeenCalled();
|
||||
expect(engine.stop).toHaveBeenCalled();
|
||||
expect(runtime.getSnapshot().phase).toBe("capturing");
|
||||
expect(engine.clearQueue).toHaveBeenCalled();
|
||||
expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(false);
|
||||
expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(true);
|
||||
});
|
||||
|
||||
it("authoritatively stops and suppresses later voice audio", async () => {
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import type { AgentStreamEventPayload } from "@server/shared/messages";
|
||||
import { Buffer } from "buffer";
|
||||
import type {
|
||||
AgentStreamEventPayload,
|
||||
SessionOutboundMessage,
|
||||
} from "@server/shared/messages";
|
||||
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
|
||||
import type { DaemonServerInfo } from "@/stores/session-store";
|
||||
import type { AudioEngine } from "@/voice/audio-engine-types";
|
||||
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
|
||||
import { SpeechSegmenter } from "@/voice/speech-segmenter";
|
||||
|
||||
const PCM_MIME_TYPE = "audio/pcm;rate=16000;bits=16";
|
||||
const KEEP_AWAKE_TAG = "paseo:voice";
|
||||
@@ -22,7 +25,6 @@ export type VoiceRuntimePhase =
|
||||
| "disabled"
|
||||
| "starting"
|
||||
| "listening"
|
||||
| "capturing"
|
||||
| "submitting"
|
||||
| "waiting"
|
||||
| "playing"
|
||||
@@ -47,11 +49,8 @@ export interface VoiceRuntimeTelemetrySnapshot {
|
||||
export interface VoiceSessionAdapter {
|
||||
serverId: string;
|
||||
setVoiceMode(enabled: boolean, agentId?: string): Promise<void>;
|
||||
sendVoiceAudioChunk(
|
||||
audioData: string,
|
||||
mimeType: string,
|
||||
isLast: boolean
|
||||
): Promise<void>;
|
||||
sendVoiceAudioChunk(audioData: string, mimeType: string): Promise<void>;
|
||||
audioPlayed(chunkId: string): Promise<void>;
|
||||
abortRequest(): Promise<void>;
|
||||
setAssistantAudioPlaying(isPlaying: boolean): void;
|
||||
}
|
||||
@@ -68,16 +67,51 @@ interface RuntimeSessionState {
|
||||
connected: boolean;
|
||||
}
|
||||
|
||||
interface ContinuousVoiceUploader {
|
||||
reset(): void;
|
||||
pushPcmChunk(chunk: Uint8Array): void;
|
||||
}
|
||||
|
||||
interface RuntimeState {
|
||||
snapshot: VoiceRuntimeSnapshot;
|
||||
telemetry: VoiceRuntimeTelemetrySnapshot;
|
||||
turnInProgress: boolean;
|
||||
serverSpeechDetected: boolean;
|
||||
transportReady: boolean;
|
||||
generation: number;
|
||||
speechInterruptTimer: ReturnType<typeof setTimeout> | null;
|
||||
speechInterruptSent: boolean;
|
||||
segmentDurationTimer: ReturnType<typeof setInterval> | null;
|
||||
lastDisplayVolumePublishMs: number;
|
||||
localSpeechStartedAt: number | null;
|
||||
}
|
||||
|
||||
type AudioOutputPayload = Extract<
|
||||
SessionOutboundMessage,
|
||||
{ type: "audio_output" }
|
||||
>["payload"];
|
||||
|
||||
type StreamingPlaybackChunk = {
|
||||
id: string;
|
||||
chunkIndex: number;
|
||||
source: { arrayBuffer(): Promise<ArrayBuffer>; size: number; type: string };
|
||||
};
|
||||
|
||||
type StreamingPlaybackGroup = {
|
||||
groupId: string;
|
||||
isVoiceMode: boolean;
|
||||
shouldPlay: boolean;
|
||||
chunks: Map<number, StreamingPlaybackChunk>;
|
||||
nextChunkToPlay: number;
|
||||
finalChunkIndex: number | null;
|
||||
started: boolean;
|
||||
ackedChunkIds: Set<string>;
|
||||
};
|
||||
|
||||
interface RuntimePlaybackState {
|
||||
groups: Map<string, StreamingPlaybackGroup>;
|
||||
orderedGroupIds: string[];
|
||||
activeGroupId: string | null;
|
||||
processing: boolean;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
|
||||
@@ -131,6 +165,7 @@ export interface VoiceRuntime {
|
||||
updateSessionConnection(serverId: string, connected: boolean): void;
|
||||
handleCapturePcm(chunk: Uint8Array): void;
|
||||
handleCaptureVolume(level: number): void;
|
||||
handleAudioOutput(serverId: string, payload: AudioOutputPayload): void;
|
||||
startVoice(serverId: string, agentId: string): Promise<void>;
|
||||
stopVoice(): Promise<void>;
|
||||
destroy(): Promise<void>;
|
||||
@@ -140,6 +175,7 @@ export interface VoiceRuntime {
|
||||
onAssistantAudioStarted(serverId: string): void;
|
||||
onAssistantAudioFinished(serverId: string): void;
|
||||
onTranscriptionResult(serverId: string, text: string): void;
|
||||
onServerSpeechStateChanged(serverId: string, isSpeaking: boolean): void;
|
||||
onTurnEvent(serverId: string, agentId: string, eventType: TurnEventType): void;
|
||||
}
|
||||
|
||||
@@ -151,12 +187,19 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
snapshot: INITIAL_SNAPSHOT,
|
||||
telemetry: INITIAL_TELEMETRY,
|
||||
turnInProgress: false,
|
||||
serverSpeechDetected: false,
|
||||
transportReady: false,
|
||||
generation: 0,
|
||||
speechInterruptTimer: null,
|
||||
speechInterruptSent: false,
|
||||
segmentDurationTimer: null,
|
||||
lastDisplayVolumePublishMs: 0,
|
||||
localSpeechStartedAt: null,
|
||||
};
|
||||
const playback: RuntimePlaybackState = {
|
||||
groups: new Map(),
|
||||
orderedGroupIds: [],
|
||||
activeGroupId: null,
|
||||
processing: false,
|
||||
generation: 0,
|
||||
};
|
||||
|
||||
function emit(): void {
|
||||
@@ -213,10 +256,151 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
return sessions.get(state.snapshot.activeServerId) ?? null;
|
||||
}
|
||||
|
||||
function clearSpeechInterruptTimer(): void {
|
||||
if (state.speechInterruptTimer) {
|
||||
clearTimeout(state.speechInterruptTimer);
|
||||
state.speechInterruptTimer = null;
|
||||
function decodeAudioChunk(base64: string): Uint8Array {
|
||||
return Buffer.from(base64, "base64");
|
||||
}
|
||||
|
||||
function toPlaybackSource(
|
||||
bytes: Uint8Array,
|
||||
format: string
|
||||
): { arrayBuffer(): Promise<ArrayBuffer>; size: number; type: string } {
|
||||
const mimeType =
|
||||
format === "pcm"
|
||||
? "audio/pcm;rate=24000;bits=16"
|
||||
: format === "mp3"
|
||||
? "audio/mpeg"
|
||||
: `audio/${format}`;
|
||||
|
||||
return {
|
||||
size: bytes.byteLength,
|
||||
type: mimeType,
|
||||
async arrayBuffer() {
|
||||
return Uint8Array.from(bytes).buffer;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function resetPlaybackState(): void {
|
||||
const hadGroups = playback.groups.size;
|
||||
playback.generation += 1;
|
||||
playback.groups.clear();
|
||||
playback.orderedGroupIds = [];
|
||||
playback.activeGroupId = null;
|
||||
playback.processing = false;
|
||||
if (hadGroups > 0) {
|
||||
console.log(`[VoiceRuntime] resetPlaybackState: cleared ${hadGroups} groups, new gen=${playback.generation}`);
|
||||
}
|
||||
}
|
||||
|
||||
function activateNextPlaybackGroup(): void {
|
||||
while (playback.orderedGroupIds.length > 0) {
|
||||
const groupId = playback.orderedGroupIds[0]!;
|
||||
if (playback.groups.has(groupId)) {
|
||||
playback.activeGroupId = groupId;
|
||||
return;
|
||||
}
|
||||
playback.orderedGroupIds.shift();
|
||||
}
|
||||
playback.activeGroupId = null;
|
||||
}
|
||||
|
||||
async function acknowledgeChunk(chunkId: string): Promise<void> {
|
||||
const activeSession = getActiveSession();
|
||||
if (!activeSession) {
|
||||
return;
|
||||
}
|
||||
await activeSession.adapter.audioPlayed(chunkId);
|
||||
}
|
||||
|
||||
async function processPlaybackQueue(serverId: string): Promise<void> {
|
||||
if (playback.processing) {
|
||||
return;
|
||||
}
|
||||
|
||||
playback.processing = true;
|
||||
const generation = playback.generation;
|
||||
console.log(`[VoiceRuntime] processPlaybackQueue start gen=${generation} activeGroup=${playback.activeGroupId}`);
|
||||
try {
|
||||
while (playback.activeGroupId) {
|
||||
if (generation !== playback.generation) {
|
||||
console.log(`[VoiceRuntime] processPlaybackQueue abort: generation changed ${generation} -> ${playback.generation}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const group = playback.groups.get(playback.activeGroupId);
|
||||
if (!group) {
|
||||
activateNextPlaybackGroup();
|
||||
continue;
|
||||
}
|
||||
|
||||
const nextChunk = group.chunks.get(group.nextChunkToPlay);
|
||||
if (!nextChunk) {
|
||||
if (
|
||||
group.finalChunkIndex !== null &&
|
||||
group.nextChunkToPlay > group.finalChunkIndex
|
||||
) {
|
||||
console.log(`[VoiceRuntime] group=${group.groupId} complete, played=${group.started} chunks=${group.nextChunkToPlay}`);
|
||||
playback.groups.delete(group.groupId);
|
||||
if (playback.orderedGroupIds[0] === group.groupId) {
|
||||
playback.orderedGroupIds.shift();
|
||||
} else {
|
||||
playback.orderedGroupIds = playback.orderedGroupIds.filter(
|
||||
(value) => value !== group.groupId
|
||||
);
|
||||
}
|
||||
if (group.started && group.isVoiceMode) {
|
||||
api.onAssistantAudioFinished(serverId);
|
||||
}
|
||||
activateNextPlaybackGroup();
|
||||
continue;
|
||||
}
|
||||
console.log(`[VoiceRuntime] group=${group.groupId} waiting for chunk=${group.nextChunkToPlay} (finalChunkIndex=${group.finalChunkIndex})`);
|
||||
return;
|
||||
}
|
||||
|
||||
group.chunks.delete(group.nextChunkToPlay);
|
||||
|
||||
if (group.shouldPlay && !group.started && group.isVoiceMode) {
|
||||
group.started = true;
|
||||
console.log(`[VoiceRuntime] group=${group.groupId} first play starting at chunk=${group.nextChunkToPlay}`);
|
||||
api.onAssistantAudioStarted(serverId);
|
||||
}
|
||||
|
||||
const playStart = Date.now();
|
||||
try {
|
||||
if (group.shouldPlay) {
|
||||
await deps.engine.play(nextChunk.source);
|
||||
console.log(`[VoiceRuntime] played chunk=${group.nextChunkToPlay} id=${nextChunk.id} took=${Date.now() - playStart}ms`);
|
||||
} else {
|
||||
console.log(`[VoiceRuntime] SKIPPED chunk=${group.nextChunkToPlay} id=${nextChunk.id} shouldPlay=false`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (generation !== playback.generation) {
|
||||
console.log(`[VoiceRuntime] play error + generation changed, aborting`);
|
||||
return;
|
||||
}
|
||||
console.error(`[VoiceRuntime] play error chunk=${group.nextChunkToPlay} took=${Date.now() - playStart}ms:`, error);
|
||||
}
|
||||
|
||||
if (generation !== playback.generation) {
|
||||
console.log(`[VoiceRuntime] post-play generation changed ${generation} -> ${playback.generation}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!group.ackedChunkIds.has(nextChunk.id)) {
|
||||
group.ackedChunkIds.add(nextChunk.id);
|
||||
void acknowledgeChunk(nextChunk.id).catch((error) => {
|
||||
console.warn("[VoiceRuntime] Failed to confirm audio playback:", error);
|
||||
});
|
||||
}
|
||||
|
||||
group.nextChunkToPlay += 1;
|
||||
}
|
||||
} finally {
|
||||
if (generation === playback.generation) {
|
||||
playback.processing = false;
|
||||
}
|
||||
console.log(`[VoiceRuntime] processPlaybackQueue exit gen=${generation} currentGen=${playback.generation}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,9 +411,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
function reconcileSegmentDurationTimer(
|
||||
segmenter: SpeechSegmenter
|
||||
): void {
|
||||
function reconcileSegmentDurationTimer(): void {
|
||||
if (!state.telemetry.isDetecting && !state.telemetry.isSpeaking) {
|
||||
clearSegmentDurationTimer();
|
||||
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
|
||||
@@ -241,7 +423,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
}
|
||||
|
||||
state.segmentDurationTimer = setInterval(() => {
|
||||
const startedAt = segmenter.getSpeechDetectionStartMs();
|
||||
const startedAt = state.localSpeechStartedAt;
|
||||
patchTelemetry((prev) => ({
|
||||
...prev,
|
||||
segmentDuration: startedAt ? Date.now() - startedAt : 0,
|
||||
@@ -262,10 +444,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
deps.engine.stopLooping();
|
||||
}
|
||||
|
||||
function resetSegmenter(segmenter: SpeechSegmenter): void {
|
||||
segmenter.reset();
|
||||
clearSpeechInterruptTimer();
|
||||
function resetCaptureTelemetry(): void {
|
||||
clearSegmentDurationTimer();
|
||||
state.localSpeechStartedAt = null;
|
||||
patchTelemetry({ ...INITIAL_TELEMETRY });
|
||||
}
|
||||
|
||||
@@ -277,134 +458,34 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
deps.engine.playLooping(new Uint8Array(0), THINKING_TONE_REPEAT_GAP_MS);
|
||||
}
|
||||
|
||||
async function interruptActiveTurn(): Promise<void> {
|
||||
const activeSession = getActiveSession();
|
||||
if (!activeSession) {
|
||||
return;
|
||||
}
|
||||
const uploader: ContinuousVoiceUploader = {
|
||||
reset() {},
|
||||
pushPcmChunk(chunk) {
|
||||
const activeSession = getActiveSession();
|
||||
if (
|
||||
!activeSession ||
|
||||
!state.transportReady ||
|
||||
!state.snapshot.isVoiceMode ||
|
||||
chunk.byteLength === 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
stopCue();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
activeSession.adapter.setAssistantAudioPlaying(false);
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "capturing" }));
|
||||
|
||||
try {
|
||||
await activeSession.adapter.abortRequest();
|
||||
} catch (error) {
|
||||
console.error("[VoiceRuntime] Failed to abort active turn:", error);
|
||||
}
|
||||
}
|
||||
|
||||
const segmenter = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming: false,
|
||||
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
|
||||
confirmedDropGracePeriodMs:
|
||||
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
|
||||
silenceDurationMs: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
|
||||
speechConfirmationMs: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
|
||||
detectionGracePeriodMs: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
|
||||
void activeSession.adapter
|
||||
.sendVoiceAudioChunk(Buffer.from(chunk).toString("base64"), PCM_MIME_TYPE)
|
||||
.catch((error) => {
|
||||
console.error("[VoiceRuntime] Failed to send audio chunk:", error);
|
||||
});
|
||||
},
|
||||
{
|
||||
onAudioSegment: ({ audioData, isLast }) => {
|
||||
const activeSession = getActiveSession();
|
||||
if (!activeSession || !state.transportReady || !state.snapshot.isVoiceMode) {
|
||||
return;
|
||||
}
|
||||
|
||||
const generation = state.generation;
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
phase: isLast ? "submitting" : "capturing",
|
||||
}));
|
||||
if (isLast) {
|
||||
state.turnInProgress = true;
|
||||
}
|
||||
|
||||
void activeSession.adapter
|
||||
.sendVoiceAudioChunk(audioData, PCM_MIME_TYPE, isLast)
|
||||
.then(() => {
|
||||
if (
|
||||
!isLast ||
|
||||
generation !== state.generation ||
|
||||
!state.snapshot.isVoiceMode ||
|
||||
state.snapshot.activeServerId !== activeSession.adapter.serverId ||
|
||||
state.snapshot.phase !== "submitting"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
|
||||
reconcileCue();
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error("[VoiceRuntime] Failed to send audio segment:", error);
|
||||
});
|
||||
},
|
||||
onSpeechStart: () => {
|
||||
stopCue();
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
clearSpeechInterruptTimer();
|
||||
state.speechInterruptSent = false;
|
||||
reconcileCue();
|
||||
},
|
||||
onDetectingChange: (isDetecting) => {
|
||||
const previous = state.telemetry;
|
||||
patchTelemetry((prev) => ({ ...prev, isDetecting }));
|
||||
if (previous.isDetecting !== isDetecting) {
|
||||
}
|
||||
if (!previous.isDetecting && isDetecting) {
|
||||
stopCue();
|
||||
}
|
||||
reconcileSegmentDurationTimer(segmenter);
|
||||
reconcileCue();
|
||||
},
|
||||
onSpeakingChange: (isSpeaking) => {
|
||||
const previous = state.telemetry;
|
||||
patchTelemetry((prev) => ({ ...prev, isSpeaking }));
|
||||
if (previous.isSpeaking !== isSpeaking) {
|
||||
}
|
||||
|
||||
if (!previous.isSpeaking && isSpeaking) {
|
||||
stopCue();
|
||||
if (
|
||||
state.snapshot.phase === "waiting" ||
|
||||
state.snapshot.phase === "playing"
|
||||
) {
|
||||
clearSpeechInterruptTimer();
|
||||
state.speechInterruptTimer = setTimeout(() => {
|
||||
state.speechInterruptTimer = null;
|
||||
if (
|
||||
state.speechInterruptSent ||
|
||||
!state.snapshot.isVoiceMode ||
|
||||
!state.telemetry.isSpeaking
|
||||
) {
|
||||
return;
|
||||
}
|
||||
state.speechInterruptSent = true;
|
||||
void interruptActiveTurn();
|
||||
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
|
||||
}
|
||||
}
|
||||
|
||||
if (!isSpeaking) {
|
||||
clearSpeechInterruptTimer();
|
||||
state.speechInterruptSent = false;
|
||||
}
|
||||
|
||||
reconcileSegmentDurationTimer(segmenter);
|
||||
reconcileCue();
|
||||
},
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
function resetToDisabledState(): void {
|
||||
state.transportReady = false;
|
||||
state.turnInProgress = false;
|
||||
state.speechInterruptSent = false;
|
||||
state.serverSpeechDetected = false;
|
||||
state.lastDisplayVolumePublishMs = 0;
|
||||
resetSegmenter(segmenter);
|
||||
uploader.reset();
|
||||
resetCaptureTelemetry();
|
||||
patchSnapshot({ ...INITIAL_SNAPSHOT });
|
||||
}
|
||||
|
||||
@@ -435,6 +516,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
|
||||
async function performLocalStop(): Promise<void> {
|
||||
stopCue();
|
||||
uploader.reset();
|
||||
resetPlaybackState();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
await deps.engine.stopCapture().catch(() => undefined);
|
||||
@@ -466,7 +549,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
const api: VoiceRuntime = {
|
||||
subscribe(listener) {
|
||||
listeners.add(listener);
|
||||
return () => {
|
||||
@@ -524,7 +607,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
|
||||
return;
|
||||
}
|
||||
segmenter.pushPcmChunk(chunk);
|
||||
uploader.pushPcmChunk(chunk);
|
||||
},
|
||||
|
||||
handleCaptureVolume(level) {
|
||||
@@ -532,9 +615,81 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
const displayLevel = state.snapshot.isMuted ? 0 : level;
|
||||
publishDisplayVolume(displayLevel, nowMs);
|
||||
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
|
||||
state.localSpeechStartedAt = null;
|
||||
patchTelemetry((prev) => ({
|
||||
...prev,
|
||||
isDetecting: false,
|
||||
isSpeaking: false,
|
||||
segmentDuration: 0,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
segmenter.pushVolumeLevel(level, nowMs);
|
||||
|
||||
const isActive = level > REALTIME_VOICE_VAD_CONFIG.volumeThreshold;
|
||||
if (isActive && state.localSpeechStartedAt === null) {
|
||||
state.localSpeechStartedAt = nowMs;
|
||||
}
|
||||
if (!isActive) {
|
||||
state.localSpeechStartedAt = null;
|
||||
}
|
||||
|
||||
patchTelemetry((prev) => ({
|
||||
...prev,
|
||||
isDetecting: isActive,
|
||||
isSpeaking: state.serverSpeechDetected,
|
||||
}));
|
||||
reconcileSegmentDurationTimer();
|
||||
if (!isActive) {
|
||||
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
|
||||
}
|
||||
reconcileCue();
|
||||
},
|
||||
|
||||
handleAudioOutput(serverId, payload) {
|
||||
if (
|
||||
serverId !== state.snapshot.activeServerId ||
|
||||
!state.snapshot.isVoiceMode ||
|
||||
!payload.isVoiceMode
|
||||
) {
|
||||
console.log(`[VoiceRuntime] audio_output DROPPED: activeServer=${state.snapshot.activeServerId} serverId=${serverId} isVoiceMode=${state.snapshot.isVoiceMode} payloadVoice=${payload.isVoiceMode}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const groupId = payload.groupId ?? payload.id;
|
||||
const chunkIndex = payload.chunkIndex ?? 0;
|
||||
console.log(`[VoiceRuntime] audio_output groupId=${groupId} chunk=${chunkIndex} isLast=${payload.isLastChunk} size=${payload.audio.length} format=${payload.format}`);
|
||||
|
||||
let group = playback.groups.get(groupId);
|
||||
if (!group) {
|
||||
const shouldPlay = api.shouldPlayVoiceAudio(serverId);
|
||||
console.log(`[VoiceRuntime] new group=${groupId} shouldPlay=${shouldPlay} phase=${state.snapshot.phase} isDetecting=${state.telemetry.isDetecting} isSpeaking=${state.telemetry.isSpeaking}`);
|
||||
group = {
|
||||
groupId,
|
||||
isVoiceMode: payload.isVoiceMode,
|
||||
shouldPlay: api.shouldPlayVoiceAudio(serverId),
|
||||
chunks: new Map(),
|
||||
nextChunkToPlay: 0,
|
||||
finalChunkIndex: null,
|
||||
started: false,
|
||||
ackedChunkIds: new Set(),
|
||||
};
|
||||
playback.groups.set(groupId, group);
|
||||
playback.orderedGroupIds.push(groupId);
|
||||
if (!playback.activeGroupId) {
|
||||
playback.activeGroupId = groupId;
|
||||
}
|
||||
}
|
||||
|
||||
group.chunks.set(chunkIndex, {
|
||||
id: payload.id,
|
||||
chunkIndex,
|
||||
source: toPlaybackSource(decodeAudioChunk(payload.audio), payload.format),
|
||||
});
|
||||
if (payload.isLastChunk) {
|
||||
group.finalChunkIndex = chunkIndex;
|
||||
}
|
||||
|
||||
void processPlaybackQueue(serverId);
|
||||
},
|
||||
|
||||
async startVoice(serverId, agentId) {
|
||||
@@ -594,7 +749,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
|
||||
state.transportReady = true;
|
||||
state.turnInProgress = false;
|
||||
resetSegmenter(segmenter);
|
||||
uploader.reset();
|
||||
resetCaptureTelemetry();
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isVoiceMode: true,
|
||||
@@ -619,8 +775,10 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
}));
|
||||
|
||||
try {
|
||||
state.transportReady = false;
|
||||
stopCue();
|
||||
uploader.reset();
|
||||
state.transportReady = false;
|
||||
resetPlaybackState();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
activeSession?.adapter.setAssistantAudioPlaying(false);
|
||||
@@ -647,7 +805,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
toggleMute() {
|
||||
const nextMuted = deps.engine.toggleMute();
|
||||
if (nextMuted) {
|
||||
resetSegmenter(segmenter);
|
||||
uploader.reset();
|
||||
resetCaptureTelemetry();
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isMuted: true,
|
||||
@@ -672,9 +831,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
state.snapshot.isVoiceMode &&
|
||||
state.snapshot.activeServerId === serverId &&
|
||||
state.snapshot.phase !== "stopping" &&
|
||||
state.snapshot.phase !== "disabled" &&
|
||||
!state.telemetry.isDetecting &&
|
||||
!state.telemetry.isSpeaking
|
||||
state.snapshot.phase !== "disabled"
|
||||
);
|
||||
},
|
||||
|
||||
@@ -701,8 +858,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
}
|
||||
|
||||
if (state.turnInProgress) {
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
|
||||
reconcileCue();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -719,6 +874,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
}
|
||||
|
||||
if (text.trim()) {
|
||||
state.turnInProgress = true;
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
|
||||
reconcileCue();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -727,6 +885,30 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
stopCue();
|
||||
},
|
||||
|
||||
onServerSpeechStateChanged(serverId, isSpeaking) {
|
||||
if (
|
||||
serverId !== state.snapshot.activeServerId ||
|
||||
!state.snapshot.isVoiceMode
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[VoiceRuntime] onServerSpeechStateChanged isSpeaking=${isSpeaking} phase=${state.snapshot.phase}`);
|
||||
state.serverSpeechDetected = isSpeaking;
|
||||
if (isSpeaking) {
|
||||
resetPlaybackState();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
getActiveSession()?.adapter.setAssistantAudioPlaying(false);
|
||||
}
|
||||
patchTelemetry((prev) => ({
|
||||
...prev,
|
||||
isSpeaking,
|
||||
}));
|
||||
reconcileSegmentDurationTimer();
|
||||
reconcileCue();
|
||||
},
|
||||
|
||||
onTurnEvent(serverId, agentId, eventType) {
|
||||
if (
|
||||
!state.snapshot.isVoiceMode ||
|
||||
@@ -738,6 +920,10 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
|
||||
if (eventType === "turn_started") {
|
||||
state.turnInProgress = true;
|
||||
if (state.snapshot.phase !== "playing") {
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
|
||||
reconcileCue();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -748,4 +934,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
stopCue();
|
||||
},
|
||||
};
|
||||
|
||||
return api;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user