From 82466aaa9fc595c9bae3fe13212fe8109747cc61 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 7 Apr 2026 18:41:46 +0700 Subject: [PATCH] Reset Silero VAD between voice turns to prevent LSTM drift The Silero VAD session ran for the entire voice mode lifetime without resetting its LSTM hidden states. Over time the internal state drifted, causing phantom speech detection on silence and getting permanently stuck in the "speaking" phase. Add reset() to TurnDetectionSession and call it after each completed utterance to clear the LSTM hidden states between turns. --- .../local/sherpa/silero-vad-session.ts | 163 +++++++++++++++--- .../server/speech/turn-detection-provider.ts | 1 + .../voice/voice-turn-controller.test.ts | 1 + .../src/server/voice/voice-turn-controller.ts | 4 +- 4 files changed, 141 insertions(+), 28 deletions(-) diff --git a/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts b/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts index c212d7204..87fe92b1f 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts @@ -8,10 +8,17 @@ import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js"; const DEFAULT_SAMPLE_RATE = 16000; const DEFAULT_BUFFER_SIZE_SECONDS = 60; const DEFAULT_SILERO_THRESHOLD = 0.5; -const DEFAULT_MIN_SILENCE_DURATION = 1.2; -const DEFAULT_MIN_SPEECH_DURATION = 0.2; const DEFAULT_WINDOW_SIZE = 512; +// Silero internal durations — kept low so isDetected() tracks actual sound. +// Our own state machine handles speech confirmation and end-of-speech detection. +const SILERO_MIN_SILENCE_DURATION = 0.2; +const SILERO_MIN_SPEECH_DURATION = 0.1; + +// Our boundary detection thresholds (in milliseconds). +const DEFAULT_CONFIRM_MS = 800; +const DEFAULT_SILENCE_MS = 1000; + type SherpaVadHandle = { acceptWaveform(samples: Float32Array): void; isDetected(): boolean; @@ -42,20 +49,30 @@ export interface SherpaSileroVadSessionConfig { modelPath?: string; sampleRate?: number; threshold?: number; - minSilenceDuration?: number; - minSpeechDuration?: number; windowSize?: number; bufferSizeInSeconds?: number; + confirmMs?: number; + silenceMs?: number; } +type VadPhase = + | { state: "idle" } + | { state: "confirming"; startedAt: number } + | { state: "speaking" } + | { state: "ending"; startedAt: number }; + export class SherpaSileroVadSession extends EventEmitter implements TurnDetectionSession { public readonly requiredSampleRate: number; private readonly vad: SherpaVadHandle; private readonly inputBuffer: SherpaCircularBufferHandle; private readonly windowSize: number; + private readonly msPerWindow: number; + private readonly confirmMs: number; + private readonly silenceMs: number; private connected = false; - private inSpeech = false; + private phase: VadPhase = { state: "idle" }; + private windowTimestamp = 0; private readonly logger; constructor(params: { @@ -67,16 +84,35 @@ export class SherpaSileroVadSession extends EventEmitter implements TurnDetectio const config = params.config ?? {}; this.requiredSampleRate = config.sampleRate ?? DEFAULT_SAMPLE_RATE; this.windowSize = config.windowSize ?? DEFAULT_WINDOW_SIZE; + this.msPerWindow = (this.windowSize / this.requiredSampleRate) * 1000; + this.confirmMs = config.confirmMs ?? DEFAULT_CONFIRM_MS; + this.silenceMs = config.silenceMs ?? DEFAULT_SILENCE_MS; + + const threshold = config.threshold ?? DEFAULT_SILERO_THRESHOLD; + + this.logger.debug( + { + threshold, + sileroMinSilenceDuration: SILERO_MIN_SILENCE_DURATION, + sileroMinSpeechDuration: SILERO_MIN_SPEECH_DURATION, + confirmMs: this.confirmMs, + silenceMs: this.silenceMs, + windowSize: this.windowSize, + msPerWindow: this.msPerWindow, + sampleRate: this.requiredSampleRate, + }, + "[VAD] Initializing Silero VAD session", + ); const sherpa = loadSherpaOnnxNode() as unknown as SherpaVadModule; this.vad = new sherpa.Vad( { sileroVad: { model: config.modelPath ?? resolveBundledSileroVadModelPath(), - threshold: config.threshold ?? DEFAULT_SILERO_THRESHOLD, - minSilenceDuration: config.minSilenceDuration ?? DEFAULT_MIN_SILENCE_DURATION, - minSpeechDuration: config.minSpeechDuration ?? DEFAULT_MIN_SPEECH_DURATION, - windowSize: config.windowSize ?? DEFAULT_WINDOW_SIZE, + threshold, + minSilenceDuration: SILERO_MIN_SILENCE_DURATION, + minSpeechDuration: SILERO_MIN_SPEECH_DURATION, + windowSize: this.windowSize, }, sampleRate: this.requiredSampleRate, numThreads: 1, @@ -106,11 +142,14 @@ export class SherpaSileroVadSession extends EventEmitter implements TurnDetectio try { const samples = pcm16leToFloat32(pcm16le, 1); this.inputBuffer.push(samples); + let windowsProcessed = 0; while (this.inputBuffer.size() > this.windowSize) { const window = this.inputBuffer.get(this.inputBuffer.head(), this.windowSize, false); this.inputBuffer.pop(this.windowSize); this.vad.acceptWaveform(window); - this.syncDetectionState(); + this.windowTimestamp += this.msPerWindow; + windowsProcessed++; + this.stepStateMachine(); } } catch (error) { this.emit("error", error instanceof Error ? error : new Error(String(error))); @@ -123,41 +162,111 @@ export class SherpaSileroVadSession extends EventEmitter implements TurnDetectio } try { + this.logger.debug( + { phase: this.phase.state }, + "[VAD] Flushing remaining audio", + ); this.vad.flush(); - this.syncDetectionState(); - if (this.inSpeech) { - this.inSpeech = false; + this.stepStateMachine(); + if (this.phase.state === "speaking" || this.phase.state === "ending") { + this.logger.debug("[VAD] Forcing speech_stopped after flush"); + this.phase = { state: "idle" }; this.emit("speech_stopped"); + } else if (this.phase.state === "confirming") { + this.logger.debug("[VAD] Discarding unconfirmed speech on flush"); + this.phase = { state: "idle" }; } } catch (error) { this.emit("error", error instanceof Error ? error : new Error(String(error))); } } - close(): void { + reset(): void { try { this.vad.reset(); this.inputBuffer.reset(); } catch { // ignore native cleanup failures } finally { - this.connected = false; - this.inSpeech = false; + this.phase = { state: "idle" }; } } - private syncDetectionState(): void { - const detected = this.vad.isDetected(); - if (detected && !this.inSpeech) { - this.inSpeech = true; - this.emit("speech_started"); - return; - } + close(): void { + this.reset(); + this.connected = false; + this.windowTimestamp = 0; + } - if (!detected && this.inSpeech && !this.vad.isEmpty()) { - this.logger.debug("Silero VAD marked end of speech"); - this.inSpeech = false; - this.emit("speech_stopped"); + private stepStateMachine(): void { + const detected = this.vad.isDetected(); + const now = this.windowTimestamp; + + switch (this.phase.state) { + case "idle": { + if (detected) { + this.logger.debug( + { now }, + "[VAD] idle → confirming (detection started)", + ); + this.phase = { state: "confirming", startedAt: now }; + } + break; + } + + case "confirming": { + if (!detected) { + const elapsed = now - this.phase.startedAt; + this.logger.debug( + { elapsed, confirmMs: this.confirmMs }, + "[VAD] confirming → idle (detection dropped before confirmation)", + ); + this.phase = { state: "idle" }; + break; + } + const elapsed = now - this.phase.startedAt; + if (elapsed >= this.confirmMs) { + this.logger.debug( + { elapsed, confirmMs: this.confirmMs }, + "[VAD] confirming → speaking (speech confirmed)", + ); + this.phase = { state: "speaking" }; + this.emit("speech_started"); + } + break; + } + + case "speaking": { + if (!detected) { + this.logger.debug( + { now }, + "[VAD] speaking → ending (silence started)", + ); + this.phase = { state: "ending", startedAt: now }; + } + break; + } + + case "ending": { + if (detected) { + this.logger.debug( + { elapsed: now - this.phase.startedAt }, + "[VAD] ending → speaking (speech resumed)", + ); + this.phase = { state: "speaking" }; + break; + } + const elapsed = now - this.phase.startedAt; + if (elapsed >= this.silenceMs) { + this.logger.debug( + { elapsed, silenceMs: this.silenceMs }, + "[VAD] ending → idle (speech stopped)", + ); + this.phase = { state: "idle" }; + this.emit("speech_stopped"); + } + break; + } } } } diff --git a/packages/server/src/server/speech/turn-detection-provider.ts b/packages/server/src/server/speech/turn-detection-provider.ts index 46fb6c386..d25501833 100644 --- a/packages/server/src/server/speech/turn-detection-provider.ts +++ b/packages/server/src/server/speech/turn-detection-provider.ts @@ -10,6 +10,7 @@ export type TurnDetectionSession = { connect(): Promise; appendPcm16(pcm16le: Buffer): void; flush(): void; + reset(): void; close(): void; on(event: "speech_started", handler: () => void): unknown; diff --git a/packages/server/src/server/voice/voice-turn-controller.test.ts b/packages/server/src/server/voice/voice-turn-controller.test.ts index 1a5446eb1..92901539b 100644 --- a/packages/server/src/server/voice/voice-turn-controller.test.ts +++ b/packages/server/src/server/voice/voice-turn-controller.test.ts @@ -19,6 +19,7 @@ class FakeTurnDetectionSession extends EventEmitter implements TurnDetectionSess } flush(): void {} + reset(): void {} close(): void {} } diff --git a/packages/server/src/server/voice/voice-turn-controller.ts b/packages/server/src/server/voice/voice-turn-controller.ts index b1eeac40e..3c2f0b76d 100644 --- a/packages/server/src/server/voice/voice-turn-controller.ts +++ b/packages/server/src/server/voice/voice-turn-controller.ts @@ -9,7 +9,7 @@ import { FixedDurationPcmRingBuffer } from "./fixed-duration-pcm-ring-buffer.js" const PCM_CHANNELS = 1; const PCM_BITS_PER_SAMPLE = 16; -const DEFAULT_PREFIX_DURATION_MS = 400; +const DEFAULT_PREFIX_DURATION_MS = 1000; type VoiceInputState = | { status: "idle" } @@ -134,6 +134,8 @@ export function createVoiceTurnController(params: { utteranceChunks = []; state = { status: "listening", rollingPrefixBytes: prefixBuffer.byteLength }; + detector.reset(); + await params.callbacks.onSpeechStopped(); params.logger.info(