jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Update files

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-11 11:04:49 +07:00
parent 002c09681d
commit 0d60588431
23 changed files with 1264 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/voice/realtime-voice-config.ts
+8 -3
View File
@@ -1,7 +1,12 @@
export const REALTIME_VOICE_VAD_CONFIG = {
volumeThreshold: 0.18,
silenceDurationMs: 2200,
// Headset mics often report lower input volume than built-in mics.
// Keep threshold low enough to avoid requiring loud speech.
volumeThreshold: 0.12,
// Short staggered grace before we visually/VAD-deactivate speaking.
// Prevents fade-out between normal intra-word pauses.
confirmedDropGracePeriodMs: 1000,
// Keep turns open longer so brief "thinking pauses" don't end speech too early.
silenceDurationMs: 2000,
speechConfirmationMs: 120,
detectionGracePeriodMs: 700,
} as const;
packages/app/src/voice/speech-segmenter.test.ts
+148
View File
@@ -68,20 +68,24 @@ describe("SpeechSegmenter", () => {
segmenter.pushVolumeLevel(0.5, t0);
segmenter.pushPcmChunk(mkPcmBytes(50));
expect(detectingChanges).toEqual([true]);
expect(speakingChanges).toEqual([true]);
expect(onSpeechStart).not.toHaveBeenCalled();
// Confirm speech.
segmenter.pushVolumeLevel(0.5, t0 + 300);
segmenter.pushPcmChunk(mkPcmBytes(50));
expect(onSpeechStart).toHaveBeenCalledTimes(1);
expect(detectingChanges).toEqual([true, false]);
expect(speakingChanges).toEqual([true]);
// End speech after silence.
segmenter.pushVolumeLevel(0.0, t0 + 301);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000);
segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000 + 2000);
expect(onSpeechEnd).toHaveBeenCalledTimes(1);
expect(detectingChanges).toEqual([true, false, true, false]);
expect(speakingChanges).toEqual([true, false]);
expect(onAudioSegment).toHaveBeenCalled();
const lastCall = onAudioSegment.mock.calls.at(-1)![0];
@@ -121,4 +125,148 @@ describe("SpeechSegmenter", () => {
expect(detectingChanges).toContain(true);
expect(onSpeechStart).toHaveBeenCalledTimes(1);
});
it("keeps utterance open after fade-out and resumes speaking during silence grace", () => {
const onAudioSegment = vi.fn();
const onSpeechStart = vi.fn();
const onSpeechEnd = vi.fn();
const speakingChanges: boolean[] = [];
const segmenter = new SpeechSegmenter(
{
enableContinuousStreaming: false,
volumeThreshold: 0.2,
silenceDurationMs: 1500,
speechConfirmationMs: 120,
detectionGracePeriodMs: 400,
minChunkDurationMs: 1000, // keep everything buffered until final flush
pcmSampleRate: 1000,
},
{
onAudioSegment,
onSpeechStart,
onSpeechEnd,
onSpeakingChange: (v) => speakingChanges.push(v),
}
);
const t0 = 30_000;
// Start + confirm speaking.
segmenter.pushVolumeLevel(0.6, t0);
segmenter.pushPcmChunk(mkPcmBytes(200));
segmenter.pushVolumeLevel(0.6, t0 + 130);
segmenter.pushPcmChunk(mkPcmBytes(200));
expect(onSpeechStart).toHaveBeenCalledTimes(1);
// Drop below threshold: meter should fade, but turn remains open.
segmenter.pushVolumeLevel(0.0, t0 + 200);
segmenter.pushPcmChunk(mkPcmBytes(200));
segmenter.pushVolumeLevel(0.0, t0 + 500);
segmenter.pushPcmChunk(mkPcmBytes(200));
// Speak again before silence timeout expires: continue same utterance.
segmenter.pushVolumeLevel(0.6, t0 + 900);
segmenter.pushPcmChunk(mkPcmBytes(200));
expect(onSpeechStart).toHaveBeenCalledTimes(1);
expect(onSpeechEnd).not.toHaveBeenCalled();
// End for real.
segmenter.pushVolumeLevel(0.0, t0 + 1000);
segmenter.pushPcmChunk(mkPcmBytes(200));
segmenter.pushVolumeLevel(0.0, t0 + 2600);
segmenter.pushVolumeLevel(0.0, t0 + 4200);
expect(speakingChanges).toEqual([true, false, true, false]);
expect(onSpeechEnd).toHaveBeenCalledTimes(1);
const finalCalls = onAudioSegment.mock.calls.filter((call) => call[0].isLast);
expect(finalCalls).toHaveLength(1);
});
it("keeps confirmed speech active above release threshold even when below start threshold", () => {
const onSpeechStart = vi.fn();
const onSpeechEnd = vi.fn();
const speakingChanges: boolean[] = [];
const segmenter = new SpeechSegmenter(
{
enableContinuousStreaming: false,
volumeThreshold: 0.3,
volumeReleaseThreshold: 0.1,
silenceDurationMs: 1200,
speechConfirmationMs: 120,
detectionGracePeriodMs: 200,
minChunkDurationMs: 1000,
pcmSampleRate: 1000,
},
{
onSpeechStart,
onSpeechEnd,
onSpeakingChange: (v) => speakingChanges.push(v),
}
);
const t0 = 40_000;
// Confirm speech.
segmenter.pushVolumeLevel(0.4, t0);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.4, t0 + 130);
segmenter.pushPcmChunk(mkPcmBytes(50));
expect(onSpeechStart).toHaveBeenCalledTimes(1);
// Stay between start and release thresholds for longer than silence timeout:
// should still be considered speaking.
segmenter.pushVolumeLevel(0.15, t0 + 400);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.15, t0 + 1000);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.15, t0 + 2000);
segmenter.pushPcmChunk(mkPcmBytes(50));
expect(onSpeechEnd).not.toHaveBeenCalled();
expect(speakingChanges.at(-1)).toBe(true);
// Now actually drop below release threshold and allow silence timeout.
segmenter.pushVolumeLevel(0.0, t0 + 2001);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.0, t0 + 3405);
segmenter.pushVolumeLevel(0.0, t0 + 4706);
expect(onSpeechEnd).toHaveBeenCalledTimes(1);
expect(speakingChanges.at(-1)).toBe(false);
});
it("ignores brief confirmed dips below release threshold without flip-flopping speaking", () => {
const speakingChanges: boolean[] = [];
const segmenter = new SpeechSegmenter(
{
enableContinuousStreaming: false,
volumeThreshold: 0.2,
volumeReleaseThreshold: 0.08,
confirmedDropGracePeriodMs: 250,
silenceDurationMs: 1500,
speechConfirmationMs: 120,
detectionGracePeriodMs: 200,
minChunkDurationMs: 1000,
pcmSampleRate: 1000,
},
{
onSpeakingChange: (v) => speakingChanges.push(v),
}
);
const t0 = 50_000;
segmenter.pushVolumeLevel(0.4, t0);
segmenter.pushVolumeLevel(0.4, t0 + 130); // confirm speaking
// Two short dips below release threshold; both shorter than debounce.
segmenter.pushVolumeLevel(0.05, t0 + 300);
segmenter.pushVolumeLevel(0.09, t0 + 500);
segmenter.pushVolumeLevel(0.05, t0 + 620);
segmenter.pushVolumeLevel(0.1, t0 + 760);
expect(speakingChanges).toEqual([true]);
});
});
packages/app/src/voice/speech-segmenter.ts
+124 -71
View File
@@ -5,6 +5,8 @@ export type SpeechSegment = { audioData: string; isLast: boolean };
export type SpeechSegmenterConfig = {
enableContinuousStreaming: boolean;
volumeThreshold: number; // 0..1
volumeReleaseThreshold?: number; // 0..1; lower threshold used to keep active speech from dropping out
confirmedDropGracePeriodMs?: number; // ms dip debounce before leaving confirmed speaking
silenceDurationMs: number;
speechConfirmationMs: number;
detectionGracePeriodMs: number;
@@ -58,6 +60,7 @@ export class SpeechSegmenter {
private speechDetectionStartMs: number | null = null;
private detectionSilenceStartMs: number | null = null;
private confirmedDropStartMs: number | null = null;
private silenceStartMs: number | null = null;
private isSpeaking = false;
private isDetecting = false;
@@ -67,8 +70,14 @@ export class SpeechSegmenter {
private minChunkBytes: number;
constructor(config: SpeechSegmenterConfig, callbacks: SpeechSegmenterCallbacks = {}) {
const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold(
config.volumeThreshold,
config.volumeReleaseThreshold
);
this.config = {
...config,
volumeReleaseThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriodMs ?? 250,
minChunkDurationMs: config.minChunkDurationMs ?? 1000,
pcmSampleRate: config.pcmSampleRate ?? 16000,
pcmChannels: config.pcmChannels ?? 1,
@@ -81,8 +90,14 @@ export class SpeechSegmenter {
}
updateConfig(next: SpeechSegmenterConfig): void {
const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold(
next.volumeThreshold,
next.volumeReleaseThreshold
);
this.config = {
...next,
volumeReleaseThreshold,
confirmedDropGracePeriodMs: next.confirmedDropGracePeriodMs ?? this.config.confirmedDropGracePeriodMs,
minChunkDurationMs: next.minChunkDurationMs ?? this.config.minChunkDurationMs,
pcmSampleRate: next.pcmSampleRate ?? this.config.pcmSampleRate,
pcmChannels: next.pcmChannels ?? this.config.pcmChannels,
@@ -107,11 +122,16 @@ export class SpeechSegmenter {
return this.isDetecting;
}
getVolumeReleaseThreshold(): number {
return this.config.volumeReleaseThreshold;
}
reset(): void {
this.audioBuffer = [];
this.bufferedBytes = 0;
this.speechDetectionStartMs = null;
this.detectionSilenceStartMs = null;
this.confirmedDropStartMs = null;
this.silenceStartMs = null;
this.isSpeaking = false;
this.isDetecting = false;
@@ -170,79 +190,103 @@ export class SpeechSegmenter {
return;
}
const speechDetected = volume > this.config.volumeThreshold;
const speechDetectedForStart = volume > this.config.volumeThreshold;
const speechDetectedForContinue = volume > this.config.volumeReleaseThreshold;
if (speechDetected && !this.isSpeaking && !this.speechConfirmed) {
// Initial detection phase.
if (this.speechDetectionStartMs === null) {
this.speechDetectionStartMs = nowMs;
this.detectionSilenceStartMs = null;
this.audioBuffer = [];
this.bufferedBytes = 0;
this.setDetecting(true);
return;
}
// Volume is back above threshold - reset grace period.
this.detectionSilenceStartMs = null;
const speechDuration = nowMs - this.speechDetectionStartMs;
if (speechDuration >= this.config.speechConfirmationMs) {
this.speechConfirmed = true;
this.silenceStartMs = null;
this.setDetecting(false);
this.setSpeaking(true);
this.callbacks.onSpeechStart?.();
}
return;
}
if (speechDetected && this.isSpeaking && this.speechConfirmed) {
// Continuing confirmed speech.
this.silenceStartMs = null;
return;
}
if (!speechDetected && !this.speechConfirmed && this.speechDetectionStartMs !== null) {
// Volume dropped during detection phase - apply grace period.
if (this.detectionSilenceStartMs === null) {
this.detectionSilenceStartMs = nowMs;
return;
}
const graceDuration = nowMs - this.detectionSilenceStartMs;
if (graceDuration >= this.config.detectionGracePeriodMs) {
// Cancel detection.
this.speechDetectionStartMs = null;
this.detectionSilenceStartMs = null;
this.audioBuffer = [];
this.bufferedBytes = 0;
this.setDetecting(false);
}
return;
}
if (!speechDetected && this.isSpeaking && this.speechConfirmed) {
// Potential speech END.
if (this.silenceStartMs === null) {
this.silenceStartMs = nowMs;
return;
}
const silenceDuration = nowMs - this.silenceStartMs;
if (silenceDuration >= this.config.silenceDurationMs) {
// Speech END confirmed.
this.speechConfirmed = false;
this.speechDetectionStartMs = null;
this.silenceStartMs = null;
this.setSpeaking(false);
this.callbacks.onSpeechEnd?.();
if (this.audioBuffer.length === 0) {
// Important: downstream expects an explicit segment boundary marker.
this.callbacks.onAudioSegment?.({ audioData: "", isLast: true });
} else {
this.flush(true);
if (!this.speechConfirmed) {
if (speechDetectedForStart) {
// Initial detection phase.
this.confirmedDropStartMs = null;
if (this.speechDetectionStartMs === null) {
this.speechDetectionStartMs = nowMs;
this.detectionSilenceStartMs = null;
this.audioBuffer = [];
this.bufferedBytes = 0;
this.setDetecting(true);
this.setSpeaking(true);
return;
}
// Volume is back above threshold - reset grace period.
this.detectionSilenceStartMs = null;
this.setSpeaking(true);
const speechDuration = nowMs - this.speechDetectionStartMs;
if (speechDuration >= this.config.speechConfirmationMs) {
this.speechConfirmed = true;
this.silenceStartMs = null;
this.setDetecting(false);
this.callbacks.onSpeechStart?.();
}
return;
}
if (this.speechDetectionStartMs !== null) {
// Volume dropped during detection phase - apply grace period.
if (this.detectionSilenceStartMs === null) {
this.detectionSilenceStartMs = nowMs;
this.setSpeaking(false);
return;
}
const graceDuration = nowMs - this.detectionSilenceStartMs;
if (graceDuration >= this.config.detectionGracePeriodMs) {
// Cancel detection.
this.speechDetectionStartMs = null;
this.detectionSilenceStartMs = null;
this.confirmedDropStartMs = null;
this.audioBuffer = [];
this.bufferedBytes = 0;
this.setDetecting(false);
this.setSpeaking(false);
}
}
return;
}
if (speechDetectedForContinue) {
// Continuing confirmed speech (or resuming during silence grace).
this.confirmedDropStartMs = null;
this.silenceStartMs = null;
this.setDetecting(false);
this.setSpeaking(true);
return;
}
if (this.confirmedDropStartMs === null) {
this.confirmedDropStartMs = nowMs;
return;
}
const confirmedDropDuration = nowMs - this.confirmedDropStartMs;
if (confirmedDropDuration < this.config.confirmedDropGracePeriodMs) {
return;
}
// Potential speech END while utterance remains open.
if (this.silenceStartMs === null) {
this.silenceStartMs = nowMs;
this.setDetecting(true);
this.setSpeaking(false);
return;
}
const silenceDuration = nowMs - this.silenceStartMs;
if (silenceDuration >= this.config.silenceDurationMs) {
// Speech END confirmed.
this.speechConfirmed = false;
this.speechDetectionStartMs = null;
this.detectionSilenceStartMs = null;
this.confirmedDropStartMs = null;
this.silenceStartMs = null;
this.setDetecting(false);
this.setSpeaking(false);
this.callbacks.onSpeechEnd?.();
if (this.audioBuffer.length === 0) {
// Important: downstream expects an explicit segment boundary marker.
this.callbacks.onAudioSegment?.({ audioData: "", isLast: true });
} else {
this.flush(true);
}
}
}
@@ -282,4 +326,13 @@ export class SpeechSegmenter {
1000;
this.minChunkBytes = Math.round(this.pcmBytesPerMs * this.config.minChunkDurationMs);
}
private resolveVolumeReleaseThreshold(
volumeThreshold: number,
configured?: number
): number {
const defaultReleaseThreshold = volumeThreshold * 0.4;
const raw = configured ?? defaultReleaseThreshold;
return Math.max(0, Math.min(raw, volumeThreshold));
}
}