Reset Silero VAD between voice turns to prevent LSTM drift

The Silero VAD session ran for the entire voice mode lifetime without
resetting its LSTM hidden states. Over time the internal state drifted,
causing phantom speech detection on silence and getting permanently
stuck in the "speaking" phase.

Add reset() to TurnDetectionSession and call it after each completed
utterance to clear the LSTM hidden states between turns.

packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts

@@ -8,10 +8,17 @@ import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js";
 const DEFAULT_SAMPLE_RATE = 16000;
 const DEFAULT_BUFFER_SIZE_SECONDS = 60;
 const DEFAULT_SILERO_THRESHOLD = 0.5;
-const DEFAULT_MIN_SILENCE_DURATION = 1.2;
-const DEFAULT_MIN_SPEECH_DURATION = 0.2;
 const DEFAULT_WINDOW_SIZE = 512;
 
+// Silero internal durations — kept low so isDetected() tracks actual sound.
+// Our own state machine handles speech confirmation and end-of-speech detection.
+const SILERO_MIN_SILENCE_DURATION = 0.2;
+const SILERO_MIN_SPEECH_DURATION = 0.1;
+
+// Our boundary detection thresholds (in milliseconds).
+const DEFAULT_CONFIRM_MS = 800;
+const DEFAULT_SILENCE_MS = 1000;
+
 type SherpaVadHandle = {
   acceptWaveform(samples: Float32Array): void;
   isDetected(): boolean;
@@ -42,20 +49,30 @@ export interface SherpaSileroVadSessionConfig {
   modelPath?: string;
   sampleRate?: number;
   threshold?: number;
-  minSilenceDuration?: number;
-  minSpeechDuration?: number;
   windowSize?: number;
   bufferSizeInSeconds?: number;
+  confirmMs?: number;
+  silenceMs?: number;
 }
 
+type VadPhase =
+  | { state: "idle" }
+  | { state: "confirming"; startedAt: number }
+  | { state: "speaking" }
+  | { state: "ending"; startedAt: number };
+
 export class SherpaSileroVadSession extends EventEmitter implements TurnDetectionSession {
   public readonly requiredSampleRate: number;
 
   private readonly vad: SherpaVadHandle;
   private readonly inputBuffer: SherpaCircularBufferHandle;
   private readonly windowSize: number;
+  private readonly msPerWindow: number;
+  private readonly confirmMs: number;
+  private readonly silenceMs: number;
   private connected = false;
-  private inSpeech = false;
+  private phase: VadPhase = { state: "idle" };
+  private windowTimestamp = 0;
   private readonly logger;
 
   constructor(params: {
@@ -67,16 +84,35 @@ export class SherpaSileroVadSession extends EventEmitter implements TurnDetectio
     const config = params.config ?? {};
     this.requiredSampleRate = config.sampleRate ?? DEFAULT_SAMPLE_RATE;
     this.windowSize = config.windowSize ?? DEFAULT_WINDOW_SIZE;
+    this.msPerWindow = (this.windowSize / this.requiredSampleRate) * 1000;
+    this.confirmMs = config.confirmMs ?? DEFAULT_CONFIRM_MS;
+    this.silenceMs = config.silenceMs ?? DEFAULT_SILENCE_MS;
+
+    const threshold = config.threshold ?? DEFAULT_SILERO_THRESHOLD;
+
+    this.logger.debug(
+      {
+        threshold,
+        sileroMinSilenceDuration: SILERO_MIN_SILENCE_DURATION,
+        sileroMinSpeechDuration: SILERO_MIN_SPEECH_DURATION,
+        confirmMs: this.confirmMs,
+        silenceMs: this.silenceMs,
+        windowSize: this.windowSize,
+        msPerWindow: this.msPerWindow,
+        sampleRate: this.requiredSampleRate,
+      },
+      "[VAD] Initializing Silero VAD session",
+    );
 
     const sherpa = loadSherpaOnnxNode() as unknown as SherpaVadModule;
     this.vad = new sherpa.Vad(
       {
         sileroVad: {
           model: config.modelPath ?? resolveBundledSileroVadModelPath(),
-          threshold: config.threshold ?? DEFAULT_SILERO_THRESHOLD,
-          minSilenceDuration: config.minSilenceDuration ?? DEFAULT_MIN_SILENCE_DURATION,
-          minSpeechDuration: config.minSpeechDuration ?? DEFAULT_MIN_SPEECH_DURATION,
-          windowSize: config.windowSize ?? DEFAULT_WINDOW_SIZE,
+          threshold,
+          minSilenceDuration: SILERO_MIN_SILENCE_DURATION,
+          minSpeechDuration: SILERO_MIN_SPEECH_DURATION,
+          windowSize: this.windowSize,
         },
         sampleRate: this.requiredSampleRate,
         numThreads: 1,
@@ -106,11 +142,14 @@ export class SherpaSileroVadSession extends EventEmitter implements TurnDetectio
     try {
       const samples = pcm16leToFloat32(pcm16le, 1);
       this.inputBuffer.push(samples);
+      let windowsProcessed = 0;
       while (this.inputBuffer.size() > this.windowSize) {
         const window = this.inputBuffer.get(this.inputBuffer.head(), this.windowSize, false);
         this.inputBuffer.pop(this.windowSize);
         this.vad.acceptWaveform(window);
-        this.syncDetectionState();
+        this.windowTimestamp += this.msPerWindow;
+        windowsProcessed++;
+        this.stepStateMachine();
       }
     } catch (error) {
       this.emit("error", error instanceof Error ? error : new Error(String(error)));
@@ -123,41 +162,111 @@ export class SherpaSileroVadSession extends EventEmitter implements TurnDetectio
     }
 
     try {
+      this.logger.debug(
+        { phase: this.phase.state },
+        "[VAD] Flushing remaining audio",
+      );
       this.vad.flush();
-      this.syncDetectionState();
-      if (this.inSpeech) {
-        this.inSpeech = false;
+      this.stepStateMachine();
+      if (this.phase.state === "speaking" || this.phase.state === "ending") {
+        this.logger.debug("[VAD] Forcing speech_stopped after flush");
+        this.phase = { state: "idle" };
         this.emit("speech_stopped");
+      } else if (this.phase.state === "confirming") {
+        this.logger.debug("[VAD] Discarding unconfirmed speech on flush");
+        this.phase = { state: "idle" };
       }
     } catch (error) {
       this.emit("error", error instanceof Error ? error : new Error(String(error)));
     }
   }
 
-  close(): void {
+  reset(): void {
     try {
       this.vad.reset();
       this.inputBuffer.reset();
     } catch {
       // ignore native cleanup failures
     } finally {
-      this.connected = false;
-      this.inSpeech = false;
+      this.phase = { state: "idle" };
     }
   }
 
-  private syncDetectionState(): void {
-    const detected = this.vad.isDetected();
-    if (detected && !this.inSpeech) {
-      this.inSpeech = true;
-      this.emit("speech_started");
-      return;
-    }
+  close(): void {
+    this.reset();
+    this.connected = false;
+    this.windowTimestamp = 0;
+  }
 
-    if (!detected && this.inSpeech && !this.vad.isEmpty()) {
-      this.logger.debug("Silero VAD marked end of speech");
-      this.inSpeech = false;
-      this.emit("speech_stopped");
+  private stepStateMachine(): void {
+    const detected = this.vad.isDetected();
+    const now = this.windowTimestamp;
+
+    switch (this.phase.state) {
+      case "idle": {
+        if (detected) {
+          this.logger.debug(
+            { now },
+            "[VAD] idle → confirming (detection started)",
+          );
+          this.phase = { state: "confirming", startedAt: now };
+        }
+        break;
+      }
+
+      case "confirming": {
+        if (!detected) {
+          const elapsed = now - this.phase.startedAt;
+          this.logger.debug(
+            { elapsed, confirmMs: this.confirmMs },
+            "[VAD] confirming → idle (detection dropped before confirmation)",
+          );
+          this.phase = { state: "idle" };
+          break;
+        }
+        const elapsed = now - this.phase.startedAt;
+        if (elapsed >= this.confirmMs) {
+          this.logger.debug(
+            { elapsed, confirmMs: this.confirmMs },
+            "[VAD] confirming → speaking (speech confirmed)",
+          );
+          this.phase = { state: "speaking" };
+          this.emit("speech_started");
+        }
+        break;
+      }
+
+      case "speaking": {
+        if (!detected) {
+          this.logger.debug(
+            { now },
+            "[VAD] speaking → ending (silence started)",
+          );
+          this.phase = { state: "ending", startedAt: now };
+        }
+        break;
+      }
+
+      case "ending": {
+        if (detected) {
+          this.logger.debug(
+            { elapsed: now - this.phase.startedAt },
+            "[VAD] ending → speaking (speech resumed)",
+          );
+          this.phase = { state: "speaking" };
+          break;
+        }
+        const elapsed = now - this.phase.startedAt;
+        if (elapsed >= this.silenceMs) {
+          this.logger.debug(
+            { elapsed, silenceMs: this.silenceMs },
+            "[VAD] ending → idle (speech stopped)",
+          );
+          this.phase = { state: "idle" };
+          this.emit("speech_stopped");
+        }
+        break;
+      }
     }
   }
 }

packages/server/src/server/speech/turn-detection-provider.ts

@@ -10,6 +10,7 @@ export type TurnDetectionSession = {
   connect(): Promise<void>;
   appendPcm16(pcm16le: Buffer): void;
   flush(): void;
+  reset(): void;
   close(): void;
 
   on(event: "speech_started", handler: () => void): unknown;

packages/server/src/server/voice/voice-turn-controller.test.ts

@@ -19,6 +19,7 @@ class FakeTurnDetectionSession extends EventEmitter implements TurnDetectionSess
   }
 
   flush(): void {}
+  reset(): void {}
   close(): void {}
 }
 

packages/server/src/server/voice/voice-turn-controller.ts

@@ -9,7 +9,7 @@ import { FixedDurationPcmRingBuffer } from "./fixed-duration-pcm-ring-buffer.js"
 
 const PCM_CHANNELS = 1;
 const PCM_BITS_PER_SAMPLE = 16;
-const DEFAULT_PREFIX_DURATION_MS = 400;
+const DEFAULT_PREFIX_DURATION_MS = 1000;
 
 type VoiceInputState =
   | { status: "idle" }
@@ -134,6 +134,8 @@ export function createVoiceTurnController(params: {
     utteranceChunks = [];
     state = { status: "listening", rollingPrefixBytes: prefixBuffer.byteLength };
 
+    detector.reset();
+
     await params.callbacks.onSpeechStopped();
 
     params.logger.info(