Fix web voice mode; share VAD/segmenting

2026-02-04 08:22:27 +07:00
parent f08efd2da6
commit 363566587e
7 changed files with 1041 additions and 266 deletions
@@ -0,0 +1,91 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { SpeechSegmenter } from "./speech-segmenter";
+
+const mkPcmBytes = (length: number): Uint8Array => {
+  const bytes = new Uint8Array(length);
+  for (let i = 0; i < length; i++) bytes[i] = i % 255;
+  return bytes;
+};
+
+describe("SpeechSegmenter", () => {
+  it("streams continuously and flushes at stop", () => {
+    const onAudioSegment = vi.fn();
+    const segmenter = new SpeechSegmenter(
+      {
+        enableContinuousStreaming: true,
+        volumeThreshold: 0.3,
+        silenceDurationMs: 2000,
+        speechConfirmationMs: 300,
+        detectionGracePeriodMs: 200,
+        minChunkDurationMs: 100, // ensure we only flush on stop
+        pcmSampleRate: 1000,
+      },
+      { onAudioSegment }
+    );
+
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    expect(onAudioSegment).not.toHaveBeenCalled();
+
+    segmenter.stop(Date.now());
+
+    expect(onAudioSegment).toHaveBeenCalledTimes(1);
+    const lastCall = onAudioSegment.mock.calls.at(-1)![0];
+    expect(lastCall.isLast).toBe(true);
+    expect(lastCall.audioData.length).toBeGreaterThan(0);
+  });
+
+  it("detects speech, calls onSpeechStart, and flushes on speech end", () => {
+    const onAudioSegment = vi.fn();
+    const onSpeechStart = vi.fn();
+    const onSpeechEnd = vi.fn();
+    const detectingChanges: boolean[] = [];
+    const speakingChanges: boolean[] = [];
+
+    const segmenter = new SpeechSegmenter(
+      {
+        enableContinuousStreaming: false,
+        volumeThreshold: 0.3,
+        silenceDurationMs: 2000,
+        speechConfirmationMs: 300,
+        detectionGracePeriodMs: 200,
+        minChunkDurationMs: 100,
+        pcmSampleRate: 1000,
+      },
+      {
+        onAudioSegment,
+        onSpeechStart,
+        onSpeechEnd,
+        onDetectingChange: (v) => detectingChanges.push(v),
+        onSpeakingChange: (v) => speakingChanges.push(v),
+      }
+    );
+
+    const t0 = 10_000;
+
+    // Start detection.
+    segmenter.pushVolumeLevel(0.5, t0);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    expect(detectingChanges).toEqual([true]);
+    expect(onSpeechStart).not.toHaveBeenCalled();
+
+    // Confirm speech.
+    segmenter.pushVolumeLevel(0.5, t0 + 300);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    expect(onSpeechStart).toHaveBeenCalledTimes(1);
+    expect(speakingChanges).toEqual([true]);
+
+    // End speech after silence.
+    segmenter.pushVolumeLevel(0.0, t0 + 301);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000);
+
+    expect(onSpeechEnd).toHaveBeenCalledTimes(1);
+    expect(speakingChanges).toEqual([true, false]);
+    expect(onAudioSegment).toHaveBeenCalled();
+    const lastCall = onAudioSegment.mock.calls.at(-1)![0];
+    expect(lastCall.isLast).toBe(true);
+    expect(lastCall.audioData.length).toBeGreaterThan(0);
+  });
+});
@@ -0,0 +1,285 @@
+import { Buffer } from "buffer";
+
+export type SpeechSegment = { audioData: string; isLast: boolean };
+
+export type SpeechSegmenterConfig = {
+  enableContinuousStreaming: boolean;
+  volumeThreshold: number; // 0..1
+  silenceDurationMs: number;
+  speechConfirmationMs: number;
+  detectionGracePeriodMs: number;
+  minChunkDurationMs?: number;
+  pcmSampleRate?: number;
+  pcmChannels?: number;
+  pcmBitsPerSample?: number;
+};
+
+export type SpeechSegmenterCallbacks = {
+  onAudioSegment?: (segment: SpeechSegment) => void;
+  onSpeechStart?: () => void;
+  onSpeechEnd?: () => void;
+  onDetectingChange?: (isDetecting: boolean) => void;
+  onSpeakingChange?: (isSpeaking: boolean) => void;
+};
+
+function uint8ArrayToBase64(bytes: Uint8Array): string {
+  // NOTE: This is performance-sensitive during continuous streaming.
+  // Buffer-backed base64 is significantly faster than manual string building.
+  try {
+    return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
+  } catch {
+    let binary = "";
+    for (let i = 0; i < bytes.length; i++) {
+      binary += String.fromCharCode(bytes[i]);
+    }
+    // eslint-disable-next-line no-restricted-globals
+    return btoa(binary);
+  }
+}
+
+function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
+  const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const arr of arrays) {
+    result.set(arr, offset);
+    offset += arr.length;
+  }
+  return result;
+}
+
+export class SpeechSegmenter {
+  private config: Required<Omit<SpeechSegmenterConfig, "minChunkDurationMs" | "pcmSampleRate" | "pcmChannels" | "pcmBitsPerSample">> &
+    Required<Pick<SpeechSegmenterConfig, "minChunkDurationMs" | "pcmSampleRate" | "pcmChannels" | "pcmBitsPerSample">>;
+  private callbacks: SpeechSegmenterCallbacks;
+
+  private audioBuffer: Uint8Array[] = [];
+  private bufferedBytes = 0;
+
+  private speechDetectionStartMs: number | null = null;
+  private detectionSilenceStartMs: number | null = null;
+  private silenceStartMs: number | null = null;
+  private isSpeaking = false;
+  private isDetecting = false;
+  private speechConfirmed = false;
+
+  private pcmBytesPerMs: number;
+  private minChunkBytes: number;
+
+  constructor(config: SpeechSegmenterConfig, callbacks: SpeechSegmenterCallbacks = {}) {
+    this.config = {
+      ...config,
+      minChunkDurationMs: config.minChunkDurationMs ?? 1000,
+      pcmSampleRate: config.pcmSampleRate ?? 16000,
+      pcmChannels: config.pcmChannels ?? 1,
+      pcmBitsPerSample: config.pcmBitsPerSample ?? 16,
+    };
+    this.callbacks = callbacks;
+    this.pcmBytesPerMs = 0;
+    this.minChunkBytes = 0;
+    this.recomputeDerived();
+  }
+
+  updateConfig(next: SpeechSegmenterConfig): void {
+    this.config = {
+      ...next,
+      minChunkDurationMs: next.minChunkDurationMs ?? this.config.minChunkDurationMs,
+      pcmSampleRate: next.pcmSampleRate ?? this.config.pcmSampleRate,
+      pcmChannels: next.pcmChannels ?? this.config.pcmChannels,
+      pcmBitsPerSample: next.pcmBitsPerSample ?? this.config.pcmBitsPerSample,
+    };
+    this.recomputeDerived();
+  }
+
+  setCallbacks(callbacks: SpeechSegmenterCallbacks): void {
+    this.callbacks = callbacks;
+  }
+
+  getSpeechDetectionStartMs(): number | null {
+    return this.speechDetectionStartMs;
+  }
+
+  getIsSpeaking(): boolean {
+    return this.isSpeaking;
+  }
+
+  getIsDetecting(): boolean {
+    return this.isDetecting;
+  }
+
+  reset(): void {
+    this.audioBuffer = [];
+    this.bufferedBytes = 0;
+    this.speechDetectionStartMs = null;
+    this.detectionSilenceStartMs = null;
+    this.silenceStartMs = null;
+    this.isSpeaking = false;
+    this.isDetecting = false;
+    this.speechConfirmed = false;
+  }
+
+  flush(isLast: boolean): void {
+    if (this.audioBuffer.length === 0) {
+      this.bufferedBytes = 0;
+      return;
+    }
+
+    const combinedBinary = concatenateUint8Arrays(this.audioBuffer);
+    const pcmBase64 = uint8ArrayToBase64(combinedBinary);
+    this.callbacks.onAudioSegment?.({ audioData: pcmBase64, isLast });
+
+    this.audioBuffer = [];
+    this.bufferedBytes = 0;
+  }
+
+  /**
+   * Called when the input stream emits a PCM16 chunk (Uint8Array of bytes).
+   * In non-continuous mode, chunk buffering is gated by the VAD state.
+   */
+  pushPcmChunk(chunk: Uint8Array): void {
+    if (chunk.length === 0) {
+      return;
+    }
+
+    if (this.config.enableContinuousStreaming) {
+      this.audioBuffer.push(chunk);
+      this.bufferedBytes += chunk.length;
+      if (this.bufferedBytes >= this.minChunkBytes) {
+        this.flush(false);
+      }
+      return;
+    }
+
+    // Buffer if we're detecting or speaking. When speech is confirmed, we also
+    // stream partial chunks to reduce latency.
+    if (this.speechDetectionStartMs !== null || this.isSpeaking) {
+      this.audioBuffer.push(chunk);
+      this.bufferedBytes += chunk.length;
+
+      if (this.speechConfirmed && this.bufferedBytes >= this.minChunkBytes) {
+        this.flush(false);
+      }
+    }
+  }
+
+  /**
+   * Called with a normalized volume level (0..1). Drives VAD transitions.
+   */
+  pushVolumeLevel(volume: number, nowMs: number): void {
+    if (this.config.enableContinuousStreaming) {
+      return;
+    }
+
+    const speechDetected = volume > this.config.volumeThreshold;
+
+    if (speechDetected && !this.isSpeaking && !this.speechConfirmed) {
+      // Initial detection phase.
+      if (this.speechDetectionStartMs === null) {
+        this.speechDetectionStartMs = nowMs;
+        this.detectionSilenceStartMs = null;
+        this.audioBuffer = [];
+        this.bufferedBytes = 0;
+        this.setDetecting(true);
+        return;
+      }
+
+      // Volume is back above threshold - reset grace period.
+      this.detectionSilenceStartMs = null;
+
+      const speechDuration = nowMs - this.speechDetectionStartMs;
+      if (speechDuration >= this.config.speechConfirmationMs) {
+        this.speechConfirmed = true;
+        this.silenceStartMs = null;
+        this.setDetecting(false);
+        this.setSpeaking(true);
+        this.callbacks.onSpeechStart?.();
+      }
+      return;
+    }
+
+    if (speechDetected && this.isSpeaking && this.speechConfirmed) {
+      // Continuing confirmed speech.
+      this.silenceStartMs = null;
+      return;
+    }
+
+    if (!speechDetected && !this.speechConfirmed && this.speechDetectionStartMs !== null) {
+      // Volume dropped during detection phase - apply grace period.
+      if (this.detectionSilenceStartMs === null) {
+        this.detectionSilenceStartMs = nowMs;
+        return;
+      }
+
+      const graceDuration = nowMs - this.detectionSilenceStartMs;
+      if (graceDuration >= this.config.detectionGracePeriodMs) {
+        // Cancel detection.
+        this.speechDetectionStartMs = null;
+        this.detectionSilenceStartMs = null;
+        this.audioBuffer = [];
+        this.bufferedBytes = 0;
+        this.setDetecting(false);
+      }
+      return;
+    }
+
+    if (!speechDetected && this.isSpeaking && this.speechConfirmed) {
+      // Potential speech END.
+      if (this.silenceStartMs === null) {
+        this.silenceStartMs = nowMs;
+        return;
+      }
+
+      const silenceDuration = nowMs - this.silenceStartMs;
+      if (silenceDuration >= this.config.silenceDurationMs) {
+        // Speech END confirmed.
+        this.speechConfirmed = false;
+        this.speechDetectionStartMs = null;
+        this.silenceStartMs = null;
+        this.setSpeaking(false);
+        this.callbacks.onSpeechEnd?.();
+        if (this.audioBuffer.length === 0) {
+          // Important: downstream expects an explicit segment boundary marker.
+          this.callbacks.onAudioSegment?.({ audioData: "", isLast: true });
+        } else {
+          this.flush(true);
+        }
+      }
+    }
+  }
+
+  stop(nowMs: number): void {
+    void nowMs;
+    if (this.config.enableContinuousStreaming) {
+      this.flush(true);
+    }
+    // Ensure observers see state clear transitions.
+    this.setDetecting(false);
+    this.setSpeaking(false);
+    this.reset();
+  }
+
+  private setDetecting(next: boolean): void {
+    if (this.isDetecting === next) {
+      return;
+    }
+    this.isDetecting = next;
+    this.callbacks.onDetectingChange?.(next);
+  }
+
+  private setSpeaking(next: boolean): void {
+    if (this.isSpeaking === next) {
+      return;
+    }
+    this.isSpeaking = next;
+    this.callbacks.onSpeakingChange?.(next);
+  }
+
+  private recomputeDerived(): void {
+    this.pcmBytesPerMs =
+      (this.config.pcmSampleRate *
+        this.config.pcmChannels *
+        (this.config.pcmBitsPerSample / 8)) /
+      1000;
+    this.minChunkBytes = Math.round(this.pcmBytesPerMs * this.config.minChunkDurationMs);
+  }
+}