jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Add speech-start grace period before voice interrupt

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-12 14:05:46 +07:00
parent 6053ee7c34
commit 5f864a4dd7
2 changed files with 106 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/contexts/voice-context.tsx
+104 -24
View File
@@ -64,38 +64,63 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
const voiceTransportReadyRef = useRef(false);
const voiceResyncInFlightRef = useRef(false);
const silenceGraceStartMsRef = useRef<number | null>(null);
const speechInterruptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const speechInterruptStartMsRef = useRef<number | null>(null);
const speechStartInterruptSentRef = useRef(false);
const isVoiceModeRef = useRef(false);
const vadStateRef = useRef<{ isDetecting: boolean; isSpeaking: boolean }>({
isDetecting: false,
isSpeaking: false,
});
const clearSpeechStartInterruptTimer = useCallback((reason: string) => {
const timer = speechInterruptTimerRef.current;
if (timer) {
clearTimeout(timer);
speechInterruptTimerRef.current = null;
const startedAt = speechInterruptStartMsRef.current;
speechInterruptStartMsRef.current = null;
if (startedAt !== null) {
console.log("[Voice] Cleared speech-start interrupt timer", {
reason,
elapsedMs: Date.now() - startedAt,
});
} else {
console.log("[Voice] Cleared speech-start interrupt timer", { reason });
}
return;
}
speechInterruptStartMsRef.current = null;
}, []);
const interruptActiveVoiceTurn = useCallback((source: string) => {
const session = realtimeSessionRef.current;
const sessionAudioPlayer = session?.audioPlayer ?? null;
const sessionClient = session?.client ?? null;
const sessionIsPlayingAudio = session?.isPlayingAudio ?? false;
if (sessionIsPlayingAudio && sessionAudioPlayer) {
if (bargeInPlaybackStopRef.current === null) {
bargeInPlaybackStopRef.current = Date.now();
}
sessionAudioPlayer.stop();
}
try {
if (sessionClient) {
void sessionClient.abortRequest().catch((error) => {
console.error("[Voice] Failed to send abort_request:", error);
});
}
console.log("[Voice] Sent abort_request before streaming audio", { source });
} catch (error) {
console.error("[Voice] Failed to send abort_request:", error);
}
}, []);
const realtimeAudio = useSpeechmaticsAudio({
onSpeechStart: () => {
console.log("[Voice] Segment started (speech confirmed)");
// Stop audio playback if playing
const session = realtimeSessionRef.current;
const sessionAudioPlayer = session?.audioPlayer ?? null;
const sessionClient = session?.client ?? null;
const sessionIsPlayingAudio = session?.isPlayingAudio ?? false;
if (sessionIsPlayingAudio && sessionAudioPlayer) {
if (bargeInPlaybackStopRef.current === null) {
bargeInPlaybackStopRef.current = Date.now();
}
sessionAudioPlayer.stop();
}
// Abort any in-flight orchestrator turn before the new speech segment streams
try {
if (sessionClient) {
void sessionClient.abortRequest().catch((error) => {
console.error("[Voice] Failed to send abort_request:", error);
});
}
console.log("[Voice] Sent abort_request before streaming audio");
} catch (error) {
console.error("[Voice] Failed to send abort_request:", error);
}
},
onSpeechEnd: () => {
const silenceMs =
@@ -108,6 +133,8 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
console.log("[Voice] Segment finalized", { silenceMs });
}
silenceGraceStartMsRef.current = null;
clearSpeechStartInterruptTimer("speech ended");
speechStartInterruptSentRef.current = false;
},
onAudioSegment: ({ audioData, isLast }) => {
if (!voiceTransportReadyRef.current) {
@@ -158,6 +185,58 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
realtimeSessionRef.current = activeSession;
}, [activeSession]);
useEffect(() => {
isVoiceModeRef.current = isVoiceMode;
}, [isVoiceMode]);
useEffect(() => {
if (!isVoiceMode) {
clearSpeechStartInterruptTimer("voice mode disabled");
speechStartInterruptSentRef.current = false;
return;
}
if (realtimeAudio.isSpeaking) {
if (
speechStartInterruptSentRef.current ||
speechInterruptTimerRef.current !== null
) {
return;
}
speechInterruptStartMsRef.current = Date.now();
speechInterruptTimerRef.current = setTimeout(() => {
speechInterruptTimerRef.current = null;
speechInterruptStartMsRef.current = null;
if (
!isVoiceModeRef.current ||
!vadStateRef.current.isSpeaking ||
speechStartInterruptSentRef.current
) {
return;
}
speechStartInterruptSentRef.current = true;
console.log("[Voice] Speech persisted beyond grace; interrupting turn", {
graceMs: REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs,
});
interruptActiveVoiceTurn("speech_start_grace_elapsed");
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
return;
}
clearSpeechStartInterruptTimer("speech stopped before grace");
}, [
clearSpeechStartInterruptTimer,
interruptActiveVoiceTurn,
isVoiceMode,
realtimeAudio.isSpeaking,
]);
useEffect(() => {
return () => {
clearSpeechStartInterruptTimer("voice provider unmounted");
};
}, [clearSpeechStartInterruptTimer]);
useEffect(() => {
const next = {
isDetecting: realtimeAudio.isDetecting,
@@ -187,6 +266,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
// Fully idle (neither detecting nor speaking).
if (!next.isDetecting && !next.isSpeaking) {
silenceGraceStartMsRef.current = null;
speechStartInterruptSentRef.current = false;
}
vadStateRef.current = next;
packages/app/src/voice/realtime-voice-config.ts
+2
View File
@@ -9,4 +9,6 @@ export const REALTIME_VOICE_VAD_CONFIG = {
silenceDurationMs: 2000,
speechConfirmationMs: 120,
detectionGracePeriodMs: 700,
// Delay speech-start interrupts to ignore transient noise triggers.
interruptGracePeriodMs: 1000,
} as const;