Fix web voice mode; share VAD/segmenting

2026-02-04 08:22:27 +07:00
parent f08efd2da6
commit 363566587e
7 changed files with 1041 additions and 266 deletions
@@ -1,15 +1,18 @@
-import { useCallback, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+
+import { SpeechSegmenter } from "@/voice/speech-segmenter";

 export interface SpeechmaticsAudioConfig {
  onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
  onSpeechStart?: () => void;
  onSpeechEnd?: () => void;
  onError?: (error: Error) => void;
+  /** When true, stream microphone PCM continuously without VAD gating. */
  enableContinuousStreaming?: boolean;
-  volumeThreshold: number;
-  silenceDuration: number;
-  speechConfirmationDuration: number;
-  detectionGracePeriod: number;
+  volumeThreshold: number; // 0-1
+  silenceDuration: number; // ms of silence before ending segment
+  speechConfirmationDuration: number; // ms of sustained speech before confirming
+  detectionGracePeriod: number; // ms grace period for volume dips during detection
 }

 export interface SpeechmaticsAudio {
@@ -24,41 +27,324 @@ export interface SpeechmaticsAudio {
  segmentDuration: number;
 }

-/**
- * Web shim for the Speechmatics two-way audio hook.
- * Real-time microphone capture is currently unsupported on web.
- */
-export function useSpeechmaticsAudio(
-  _config: SpeechmaticsAudioConfig
-): SpeechmaticsAudio {
+const getAudioContextCtor = (): (typeof AudioContext) | null => {
+  if (typeof window === "undefined") {
+    return null;
+  }
+  const ctor =
+    (window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext ||
+    (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
+  return ctor ?? null;
+};
+
+const floatToInt16 = (sample: number): number => {
+  const clamped = Math.max(-1, Math.min(1, sample));
+  return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
+};
+
+const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
+  if (input.length === 0) {
+    return new Int16Array(0);
+  }
+  if (inputRate === outputRate) {
+    const out = new Int16Array(input.length);
+    for (let i = 0; i < input.length; i++) {
+      out[i] = floatToInt16(input[i]);
+    }
+    return out;
+  }
+
+  const ratio = inputRate / outputRate;
+  const outputLength = Math.max(1, Math.round(input.length / ratio));
+  const out = new Int16Array(outputLength);
+  for (let i = 0; i < outputLength; i++) {
+    const sourceIndex = i * ratio;
+    const i0 = Math.floor(sourceIndex);
+    const i1 = Math.min(input.length - 1, i0 + 1);
+    const frac = sourceIndex - i0;
+    const sample = input[i0] * (1 - frac) + input[i1] * frac;
+    out[i] = floatToInt16(sample);
+  }
+  return out;
+};
+
+export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): SpeechmaticsAudio {
  const [isActive, setIsActive] = useState(false);
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const [isDetecting, setIsDetecting] = useState(false);
+  const [volume, setVolume] = useState(0);
  const [isMuted, setIsMuted] = useState(false);
+  const [segmentDuration, setSegmentDuration] = useState(0);
+
+  const enableContinuousStreaming = config.enableContinuousStreaming === true;
+
+  const callbacksRef = useRef({
+    onAudioSegment: config.onAudioSegment,
+    onSpeechStart: config.onSpeechStart,
+    onSpeechEnd: config.onSpeechEnd,
+    onError: config.onError,
+  });
+
+  useEffect(() => {
+    callbacksRef.current = {
+      onAudioSegment: config.onAudioSegment,
+      onSpeechStart: config.onSpeechStart,
+      onSpeechEnd: config.onSpeechEnd,
+      onError: config.onError,
+    };
+  }, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd, config.onError]);
+
+  const segmenterRef = useRef<SpeechSegmenter | null>(null);
+  if (segmenterRef.current === null) {
+    segmenterRef.current = new SpeechSegmenter(
+      {
+        enableContinuousStreaming,
+        volumeThreshold: config.volumeThreshold,
+        silenceDurationMs: config.silenceDuration,
+        speechConfirmationMs: config.speechConfirmationDuration,
+        detectionGracePeriodMs: config.detectionGracePeriod,
+      },
+      {
+        onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
+        onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
+        onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
+        onDetectingChange: (next) => setIsDetecting(next),
+        onSpeakingChange: (next) => setIsSpeaking(next),
+      }
+    );
+  }
+
+  useEffect(() => {
+    segmenterRef.current?.updateConfig({
+      enableContinuousStreaming,
+      volumeThreshold: config.volumeThreshold,
+      silenceDurationMs: config.silenceDuration,
+      speechConfirmationMs: config.speechConfirmationDuration,
+      detectionGracePeriodMs: config.detectionGracePeriod,
+    });
+  }, [
+    enableContinuousStreaming,
+    config.volumeThreshold,
+    config.silenceDuration,
+    config.speechConfirmationDuration,
+    config.detectionGracePeriod,
+  ]);
+
+  const refs = useRef<{
+    started: boolean;
+    stream: MediaStream | null;
+    context: AudioContext | null;
+    source: MediaStreamAudioSourceNode | null;
+    processor: ScriptProcessorNode | null;
+    gain: GainNode | null;
+  }>({
+    started: false,
+    stream: null,
+    context: null,
+    source: null,
+    processor: null,
+    gain: null,
+  });
+
+  const isMutedRef = useRef(isMuted);
+  useEffect(() => {
+    isMutedRef.current = isMuted;
+  }, [isMuted]);
+
+  // Update segment duration timer
+  useEffect(() => {
+    if (!isDetecting && !isSpeaking) {
+      setSegmentDuration(0);
+      return;
+    }
+
+    const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
+    const interval = setInterval(() => {
+      setSegmentDuration(Date.now() - startTime);
+    }, 100);
+
+    return () => clearInterval(interval);
+  }, [isDetecting, isSpeaking]);
+
+  const stopInternal = useCallback(async () => {
+    refs.current.started = false;
+
+    try {
+      refs.current.processor?.disconnect();
+      refs.current.source?.disconnect();
+      refs.current.gain?.disconnect();
+    } catch {
+      // best-effort teardown
+    }
+
+    if (refs.current.stream) {
+      for (const track of refs.current.stream.getTracks()) {
+        try {
+          track.stop();
+        } catch {
+          // ignore
+        }
+      }
+    }
+
+    const context = refs.current.context;
+    if (context && context.state !== "closed") {
+      try {
+        await context.close();
+      } catch {
+        // ignore
+      }
+    }
+
+    refs.current.stream = null;
+    refs.current.context = null;
+    refs.current.source = null;
+    refs.current.processor = null;
+    refs.current.gain = null;
+
+    segmenterRef.current?.stop(Date.now());
+    setIsActive(false);
+    setIsSpeaking(false);
+    setIsDetecting(false);
+    setVolume(0);
+    setIsMuted(false);
+  }, []);

  const start = useCallback(async () => {
-    console.warn("[SpeechmaticsAudio] Web microphone capture disabled");
-    setIsActive(true);
-  }, []);
+    if (refs.current.started) {
+      return;
+    }
+
+    const missingNavigator =
+      typeof navigator === "undefined" ||
+      !navigator.mediaDevices ||
+      typeof navigator.mediaDevices.getUserMedia !== "function";
+
+    const secureContext =
+      typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
+        ? window.isSecureContext
+        : true;
+    const currentOrigin = typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
+
+    try {
+      if (missingNavigator) {
+        throw new Error("Microphone capture is not supported in this environment");
+      }
+      if (!secureContext) {
+        throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
+      }
+
+      const AudioContextCtor = getAudioContextCtor();
+      if (!AudioContextCtor) {
+        throw new Error("AudioContext unavailable");
+      }
+
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          channelCount: 1,
+          noiseSuppression: true,
+          echoCancellation: true,
+          autoGainControl: true,
+        },
+      });
+
+      const context = new AudioContextCtor();
+      if (context.state === "suspended") {
+        await context.resume();
+      }
+
+      const source = context.createMediaStreamSource(stream);
+      const processor = context.createScriptProcessor(4096, 1, 1);
+      const gain = context.createGain();
+      gain.gain.value = 0;
+
+      refs.current = {
+        started: true,
+        stream,
+        context,
+        source,
+        processor,
+        gain,
+      };
+
+      processor.onaudioprocess = (event) => {
+        if (!refs.current.started) {
+          return;
+        }
+
+        const input = event.inputBuffer.getChannelData(0);
+        let sumSquares = 0;
+        for (let i = 0; i < input.length; i++) {
+          const sample = input[i];
+          sumSquares += sample * sample;
+        }
+        const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
+        const normalized = Math.min(1, Math.max(0, rms * 2));
+        setVolume(normalized);
+
+        if (isMutedRef.current) {
+          return;
+        }
+
+        const nowMs = Date.now();
+        segmenterRef.current?.pushVolumeLevel(normalized, nowMs);
+
+        const pcm16 = resampleToPcm16(input, context.sampleRate, 16000);
+        const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength);
+        segmenterRef.current?.pushPcmChunk(pcmBytes);
+      };
+
+      source.connect(processor);
+      processor.connect(gain);
+      gain.connect(context.destination);
+
+      setIsActive(true);
+      return;
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error));
+      callbacksRef.current.onError?.(err);
+      await stopInternal();
+      throw err;
+    }
+  }, [stopInternal]);

  const stop = useCallback(async () => {
-    setIsActive(false);
-  }, []);
+    await stopInternal();
+  }, [stopInternal]);

  const toggleMute = useCallback(() => {
-    setIsMuted((prev) => !prev);
+    setIsMuted((prev) => {
+      const nextMuted = !prev;
+      if (nextMuted) {
+        segmenterRef.current?.reset();
+        setIsSpeaking(false);
+        setIsDetecting(false);
+      }
+      return nextMuted;
+    });
  }, []);

+  useEffect(() => {
+    return () => {
+      if (refs.current.started) {
+        void stopInternal();
+      }
+    };
+  }, [stopInternal]);
+
  return useMemo(
    () => ({
      start,
      stop,
      toggleMute,
      isActive,
-      isSpeaking: false,
-      isDetecting: false,
+      isSpeaking,
+      isDetecting,
      isMuted,
-      volume: 0,
-      segmentDuration: 0,
+      volume,
+      segmentDuration,
    }),
-    [start, stop, toggleMute, isActive, isMuted]
+    [start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration]
  );
 }
+