import { useState, useEffect, useRef, useCallback } from "react"; import { initialize, useMicrophonePermissions, toggleRecording, tearDown, useExpoTwoWayAudioEventListener, type MicrophoneDataCallback, type VolumeLevelCallback, } from "@boudra/expo-two-way-audio"; import { SpeechSegmenter } from "@/voice/speech-segmenter"; export interface SpeechmaticsAudioConfig { onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void; onSpeechStart?: () => void; onSpeechEnd?: () => void; onError?: (error: Error) => void; /** When true, stream microphone PCM continuously without VAD gating. */ enableContinuousStreaming?: boolean; volumeThreshold: number; // Volume threshold for speech detection (0-1) /** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */ confirmedDropGracePeriod?: number; silenceDuration: number; // ms of silence before ending segment speechConfirmationDuration: number; // ms of sustained speech before confirming detectionGracePeriod: number; // ms grace period for volume dips during detection } export interface SpeechmaticsAudio { start: () => Promise; stop: () => Promise; toggleMute: () => void; isActive: boolean; isSpeaking: boolean; isDetecting: boolean; isMuted: boolean; volume: number; segmentDuration: number; } /** * Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio */ export function useSpeechmaticsAudio( config: SpeechmaticsAudioConfig ): SpeechmaticsAudio { const [microphonePermission, requestMicrophonePermission] = useMicrophonePermissions(); const [isActive, setIsActive] = useState(false); const [isSpeaking, setIsSpeaking] = useState(false); const [isDetecting, setIsDetecting] = useState(false); const [audioInitialized, setAudioInitialized] = useState(false); const [volume, setVolume] = useState(0); const [isMuted, setIsMuted] = useState(false); const [segmentDuration, setSegmentDuration] = useState(0); const enableContinuousStreaming = config.enableContinuousStreaming === true; const isActiveRef = useRef(isActive); useEffect(() => { isActiveRef.current = isActive; }, [isActive]); const isMutedRef = useRef(isMuted); useEffect(() => { isMutedRef.current = isMuted; }, [isMuted]); const callbacksRef = useRef({ onAudioSegment: config.onAudioSegment, onSpeechStart: config.onSpeechStart, onSpeechEnd: config.onSpeechEnd, }); useEffect(() => { callbacksRef.current = { onAudioSegment: config.onAudioSegment, onSpeechStart: config.onSpeechStart, onSpeechEnd: config.onSpeechEnd, }; }, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd]); const segmenterRef = useRef(null); if (segmenterRef.current === null) { segmenterRef.current = new SpeechSegmenter( { enableContinuousStreaming, volumeThreshold: config.volumeThreshold, confirmedDropGracePeriodMs: config.confirmedDropGracePeriod, silenceDurationMs: config.silenceDuration, speechConfirmationMs: config.speechConfirmationDuration, detectionGracePeriodMs: config.detectionGracePeriod, }, { onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment), onSpeechStart: () => callbacksRef.current.onSpeechStart?.(), onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(), onDetectingChange: (next) => setIsDetecting(next), onSpeakingChange: (next) => setIsSpeaking(next), } ); } useEffect(() => { segmenterRef.current?.updateConfig({ enableContinuousStreaming, volumeThreshold: config.volumeThreshold, confirmedDropGracePeriodMs: config.confirmedDropGracePeriod, silenceDurationMs: config.silenceDuration, speechConfirmationMs: config.speechConfirmationDuration, detectionGracePeriodMs: config.detectionGracePeriod, }); }, [ enableContinuousStreaming, config.volumeThreshold, config.confirmedDropGracePeriod, config.silenceDuration, config.speechConfirmationDuration, config.detectionGracePeriod, ]); // Update segment duration timer useEffect(() => { if (!isDetecting && !isSpeaking) { setSegmentDuration(0); return; } const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now(); const interval = setInterval(() => { const elapsed = Date.now() - startTime; setSegmentDuration(elapsed); }, 100); return () => clearInterval(interval); }, [isDetecting, isSpeaking]); // Listen to microphone data useExpoTwoWayAudioEventListener( "onMicrophoneData", useCallback( (event) => { if (!isActiveRef.current || isMutedRef.current) return; const pcmData: Uint8Array = event.data; segmenterRef.current?.pushPcmChunk(pcmData); }, [] ) ); // Listen to volume level for VAD useExpoTwoWayAudioEventListener( "onInputVolumeLevelData", useCallback( (event) => { if (!isActiveRef.current) return; const volumeLevel: number = event.data; setVolume(volumeLevel); if (isMutedRef.current) return; segmenterRef.current?.pushVolumeLevel(volumeLevel, Date.now()); }, [] ) ); const ensureMicrophonePermission = useCallback(async () => { let permissionStatus = microphonePermission; if (!permissionStatus?.granted) { try { permissionStatus = await requestMicrophonePermission(); } catch (err) { throw new Error("Failed to request microphone permission"); } } if (!permissionStatus?.granted) { throw new Error( "Microphone permission is required to capture audio. Please enable microphone access in system settings." ); } }, [microphonePermission, requestMicrophonePermission]); async function start(): Promise { if (isActive) { console.log("[SpeechmaticsAudio] Already active"); return; } try { await ensureMicrophonePermission(); // Initialize audio if not already initialized if (!audioInitialized) { console.log("[SpeechmaticsAudio] Initializing audio..."); await initialize(); setAudioInitialized(true); console.log("[SpeechmaticsAudio] Audio initialized"); } console.log("[SpeechmaticsAudio] Starting audio capture..."); // Start recording toggleRecording(true); setIsActive(true); console.log("[SpeechmaticsAudio] Audio capture started successfully"); } catch (error) { console.error("[SpeechmaticsAudio] Start error:", error); const err = error instanceof Error ? error : new Error(String(error)); config.onError?.(err); await stop(); throw error; } } async function stop(): Promise { console.log("[SpeechmaticsAudio] Stopping audio capture..."); // Stop recording if (isActive) { toggleRecording(false); } segmenterRef.current?.stop(Date.now()); // Tear down audio session if (audioInitialized) { tearDown(); setAudioInitialized(false); console.log("[SpeechmaticsAudio] Audio torn down"); } // Reset state segmenterRef.current?.reset(); setIsActive(false); setIsSpeaking(false); setIsDetecting(false); setVolume(0); setIsMuted(false); console.log("[SpeechmaticsAudio] Audio capture stopped"); } function toggleMute(): void { setIsMuted((prev) => { const newMuted = !prev; console.log("[SpeechmaticsAudio] Mute toggled:", newMuted); if (newMuted) { // Clear any ongoing speech detection/speaking state segmenterRef.current?.reset(); setIsSpeaking(false); setIsDetecting(false); } return newMuted; }); } return { start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration, }; }