diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index c27c815e3..6d7995e9b 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -64,38 +64,63 @@ export function VoiceProvider({ children }: VoiceProviderProps) { const voiceTransportReadyRef = useRef(false); const voiceResyncInFlightRef = useRef(false); const silenceGraceStartMsRef = useRef(null); + const speechInterruptTimerRef = useRef | null>(null); + const speechInterruptStartMsRef = useRef(null); + const speechStartInterruptSentRef = useRef(false); + const isVoiceModeRef = useRef(false); const vadStateRef = useRef<{ isDetecting: boolean; isSpeaking: boolean }>({ isDetecting: false, isSpeaking: false, }); + const clearSpeechStartInterruptTimer = useCallback((reason: string) => { + const timer = speechInterruptTimerRef.current; + if (timer) { + clearTimeout(timer); + speechInterruptTimerRef.current = null; + const startedAt = speechInterruptStartMsRef.current; + speechInterruptStartMsRef.current = null; + if (startedAt !== null) { + console.log("[Voice] Cleared speech-start interrupt timer", { + reason, + elapsedMs: Date.now() - startedAt, + }); + } else { + console.log("[Voice] Cleared speech-start interrupt timer", { reason }); + } + return; + } + speechInterruptStartMsRef.current = null; + }, []); + + const interruptActiveVoiceTurn = useCallback((source: string) => { + const session = realtimeSessionRef.current; + const sessionAudioPlayer = session?.audioPlayer ?? null; + const sessionClient = session?.client ?? null; + const sessionIsPlayingAudio = session?.isPlayingAudio ?? false; + + if (sessionIsPlayingAudio && sessionAudioPlayer) { + if (bargeInPlaybackStopRef.current === null) { + bargeInPlaybackStopRef.current = Date.now(); + } + sessionAudioPlayer.stop(); + } + + try { + if (sessionClient) { + void sessionClient.abortRequest().catch((error) => { + console.error("[Voice] Failed to send abort_request:", error); + }); + } + console.log("[Voice] Sent abort_request before streaming audio", { source }); + } catch (error) { + console.error("[Voice] Failed to send abort_request:", error); + } + }, []); + const realtimeAudio = useSpeechmaticsAudio({ onSpeechStart: () => { console.log("[Voice] Segment started (speech confirmed)"); - // Stop audio playback if playing - const session = realtimeSessionRef.current; - const sessionAudioPlayer = session?.audioPlayer ?? null; - const sessionClient = session?.client ?? null; - const sessionIsPlayingAudio = session?.isPlayingAudio ?? false; - - if (sessionIsPlayingAudio && sessionAudioPlayer) { - if (bargeInPlaybackStopRef.current === null) { - bargeInPlaybackStopRef.current = Date.now(); - } - sessionAudioPlayer.stop(); - } - - // Abort any in-flight orchestrator turn before the new speech segment streams - try { - if (sessionClient) { - void sessionClient.abortRequest().catch((error) => { - console.error("[Voice] Failed to send abort_request:", error); - }); - } - console.log("[Voice] Sent abort_request before streaming audio"); - } catch (error) { - console.error("[Voice] Failed to send abort_request:", error); - } }, onSpeechEnd: () => { const silenceMs = @@ -108,6 +133,8 @@ export function VoiceProvider({ children }: VoiceProviderProps) { console.log("[Voice] Segment finalized", { silenceMs }); } silenceGraceStartMsRef.current = null; + clearSpeechStartInterruptTimer("speech ended"); + speechStartInterruptSentRef.current = false; }, onAudioSegment: ({ audioData, isLast }) => { if (!voiceTransportReadyRef.current) { @@ -158,6 +185,58 @@ export function VoiceProvider({ children }: VoiceProviderProps) { realtimeSessionRef.current = activeSession; }, [activeSession]); + useEffect(() => { + isVoiceModeRef.current = isVoiceMode; + }, [isVoiceMode]); + + useEffect(() => { + if (!isVoiceMode) { + clearSpeechStartInterruptTimer("voice mode disabled"); + speechStartInterruptSentRef.current = false; + return; + } + + if (realtimeAudio.isSpeaking) { + if ( + speechStartInterruptSentRef.current || + speechInterruptTimerRef.current !== null + ) { + return; + } + speechInterruptStartMsRef.current = Date.now(); + speechInterruptTimerRef.current = setTimeout(() => { + speechInterruptTimerRef.current = null; + speechInterruptStartMsRef.current = null; + if ( + !isVoiceModeRef.current || + !vadStateRef.current.isSpeaking || + speechStartInterruptSentRef.current + ) { + return; + } + speechStartInterruptSentRef.current = true; + console.log("[Voice] Speech persisted beyond grace; interrupting turn", { + graceMs: REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs, + }); + interruptActiveVoiceTurn("speech_start_grace_elapsed"); + }, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs); + return; + } + + clearSpeechStartInterruptTimer("speech stopped before grace"); + }, [ + clearSpeechStartInterruptTimer, + interruptActiveVoiceTurn, + isVoiceMode, + realtimeAudio.isSpeaking, + ]); + + useEffect(() => { + return () => { + clearSpeechStartInterruptTimer("voice provider unmounted"); + }; + }, [clearSpeechStartInterruptTimer]); + useEffect(() => { const next = { isDetecting: realtimeAudio.isDetecting, @@ -187,6 +266,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) { // Fully idle (neither detecting nor speaking). if (!next.isDetecting && !next.isSpeaking) { silenceGraceStartMsRef.current = null; + speechStartInterruptSentRef.current = false; } vadStateRef.current = next; diff --git a/packages/app/src/voice/realtime-voice-config.ts b/packages/app/src/voice/realtime-voice-config.ts index 95d866ac9..698c80cbe 100644 --- a/packages/app/src/voice/realtime-voice-config.ts +++ b/packages/app/src/voice/realtime-voice-config.ts @@ -9,4 +9,6 @@ export const REALTIME_VOICE_VAD_CONFIG = { silenceDurationMs: 2000, speechConfirmationMs: 120, detectionGracePeriodMs: 700, + // Delay speech-start interrupts to ignore transient noise triggers. + interruptGracePeriodMs: 1000, } as const;