From f08efd2da6a2f61824d75e3ded49ee7180ef0626 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 4 Feb 2026 00:03:28 +0700 Subject: [PATCH 1/2] feat: add expand/collapse all button to git diff pane --- packages/app/src/components/git-diff-pane.tsx | 96 ++++++++++++++----- 1 file changed, 71 insertions(+), 25 deletions(-) diff --git a/packages/app/src/components/git-diff-pane.tsx b/packages/app/src/components/git-diff-pane.tsx index e945e2ce6..a3a1e380c 100644 --- a/packages/app/src/components/git-diff-pane.tsx +++ b/packages/app/src/components/git-diff-pane.tsx @@ -16,7 +16,7 @@ import { ScrollView, type ScrollView as ScrollViewType } from "react-native-gest import { StyleSheet, useUnistyles } from "react-native-unistyles"; import AsyncStorage from "@react-native-async-storage/async-storage"; import * as Linking from "expo-linking"; -import { ChevronDown, ChevronRight, GitBranch, MoreVertical, ArrowLeftRight } from "lucide-react-native"; +import { ChevronDown, ChevronRight, GitBranch, MoreVertical, ArrowLeftRight, ListChevronsDownUp, ListChevronsUpDown } from "lucide-react-native"; import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useSessionStore } from "@/stores/session-store"; import { @@ -583,6 +583,23 @@ export function GitDiffPane({ serverId, agentId, cwd }: GitDiffPaneProps) { })); }, []); + const allExpanded = useMemo(() => { + if (files.length === 0) return false; + return files.every((file) => expandedByPath[file.path]); + }, [files, expandedByPath]); + + const handleToggleExpandAll = useCallback(() => { + if (allExpanded) { + setExpandedByPath({}); + } else { + const newExpanded: Record = {}; + for (const file of files) { + newExpanded[file.path] = true; + } + setExpandedByPath(newExpanded); + } + }, [allExpanded, files]); + // Reset manual refresh flag when fetch completes useEffect(() => { if (!(isDiffFetching || isStatusFetching) && isManualRefresh) { @@ -1157,27 +1174,42 @@ export function GitDiffPane({ serverId, agentId, cwd }: GitDiffPaneProps) { {isGit && (hasUncommittedChanges || aheadCount > 0) ? ( - [ - styles.diffStatusRow, - (hovered || pressed) && styles.diffStatusRowHovered, - ]} - testID="changes-diff-status" - onPress={() => setDiffModeOverride(diffMode === "uncommitted" ? "base" : "uncommitted")} - > - {({ hovered, pressed }) => ( - <> - - {diffMode === "uncommitted" ? "Uncommitted" : "Committed"} - - - + + [ + styles.diffStatusRow, + (hovered || pressed) && styles.diffStatusRowHovered, + ]} + testID="changes-diff-status" + onPress={() => setDiffModeOverride(diffMode === "uncommitted" ? "base" : "uncommitted")} + > + {({ hovered, pressed }) => ( + <> + + {diffMode === "uncommitted" ? "Uncommitted" : "Committed"} + + {(hovered || pressed) ? ( + + ) : null} + )} - + + {files.length > 0 ? ( + [ + styles.expandAllButton, + (hovered || pressed) && styles.diffStatusRowHovered, + ]} + onPress={handleToggleExpandAll} + > + {allExpanded ? ( + + ) : ( + + )} + + ) : null} + ) : null} @@ -1229,15 +1261,20 @@ const styles = StyleSheet.create((theme) => ({ borderBottomWidth: 1, borderBottomColor: theme.colors.border, }, + diffStatusInner: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + paddingRight: theme.spacing[3], + }, diffStatusRow: { flexDirection: "row", alignItems: "center", - alignSelf: "flex-start", gap: theme.spacing[1], - // Align text with chevron/branch icons (at spacing[2] from edge) + // Align text with header branch icon (at spacing[3] from edge, minus our horizontal padding) + marginLeft: theme.spacing[3] - theme.spacing[1], marginVertical: theme.spacing[2], - paddingLeft: theme.spacing[2], - paddingRight: theme.spacing[2], + paddingHorizontal: theme.spacing[1], paddingVertical: theme.spacing[1], borderRadius: theme.borderRadius.base, }, @@ -1251,6 +1288,15 @@ const styles = StyleSheet.create((theme) => ({ diffStatusIconHidden: { opacity: 0, }, + expandAllButton: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[1], + marginVertical: theme.spacing[2], + paddingHorizontal: theme.spacing[1], + paddingVertical: theme.spacing[1], + borderRadius: theme.borderRadius.base, + }, splitButton: { flexDirection: "row", alignItems: "stretch", From 363566587e902fa4989bb0041f6532b76c276496 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 4 Feb 2026 08:22:13 +0700 Subject: [PATCH 2/2] Fix web voice mode; share VAD/segmenting --- packages/app/src/contexts/voice-context.tsx | 12 +- .../app/src/hooks/use-audio-player.native.ts | 21 ++ .../app/src/hooks/use-audio-player.web.ts | 274 ++++++++++++++- .../hooks/use-speechmatics-audio.native.ts | 292 ++++----------- .../src/hooks/use-speechmatics-audio.web.ts | 332 ++++++++++++++++-- .../app/src/voice/speech-segmenter.test.ts | 91 +++++ packages/app/src/voice/speech-segmenter.ts | 285 +++++++++++++++ 7 files changed, 1041 insertions(+), 266 deletions(-) create mode 100644 packages/app/src/voice/speech-segmenter.test.ts create mode 100644 packages/app/src/voice/speech-segmenter.ts diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index 6afce3a7e..87220f1f5 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -129,11 +129,11 @@ export function VoiceProvider({ children }: VoiceProviderProps) { // Update voice detection flags whenever they change useEffect(() => { - const session = realtimeSessionRef.current; - if (session?.client) { - // Note: voice detection updates need backend support before sending. - } - }, [realtimeAudio.isDetecting, realtimeAudio.isSpeaking]); + activeSession?.methods?.setVoiceDetectionFlags( + realtimeAudio.isDetecting, + realtimeAudio.isSpeaking + ); + }, [activeSession?.methods, realtimeAudio.isDetecting, realtimeAudio.isSpeaking]); useEffect(() => { realtimeSessionRef.current = activeSession; @@ -163,6 +163,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) { try { realtimeSessionRef.current = session; setActiveServerId(serverId); + await session.audioPlayer?.warmup?.(); await realtimeAudio.start(); setIsVoiceMode(true); console.log("[Voice] Mode enabled"); @@ -195,6 +196,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) { const session = realtimeSessionRef.current; session?.audioPlayer?.stop(); await realtimeAudio.stop(); + session?.methods?.setVoiceDetectionFlags(false, false); setIsVoiceMode(false); setActiveServerId(null); console.log("[Voice] Mode disabled"); diff --git a/packages/app/src/hooks/use-audio-player.native.ts b/packages/app/src/hooks/use-audio-player.native.ts index 642d212ba..4729c9b9c 100644 --- a/packages/app/src/hooks/use-audio-player.native.ts +++ b/packages/app/src/hooks/use-audio-player.native.ts @@ -56,6 +56,10 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { const queueRef = useRef([]); const suppressedQueueRef = useRef([]); const isProcessingQueueRef = useRef(false); + const activePlaybackRef = useRef<{ + resolve: (duration: number) => void; + reject: (error: Error) => void; + } | null>(null); const playbackTimeoutRef = useRef | null>(null); const checkIntervalRef = useRef | null>(null); @@ -179,6 +183,7 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { async function playAudio(audioData: Blob): Promise { return new Promise(async (resolve, reject) => { + activePlaybackRef.current = { resolve, reject }; try { console.log( `[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})` @@ -234,6 +239,7 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { console.log("[AudioPlayer] ✅ Playback finished"); setIsPlaying(false); playbackTimeoutRef.current = null; + activePlaybackRef.current = null; resolve(durationSec); }, durationSec * 1000); } catch (error) { @@ -246,6 +252,7 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { } setIsPlaying(false); + activePlaybackRef.current = null; reject(error); } }); @@ -267,6 +274,12 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { setIsPlaying(false); + // Reject the currently playing promise, if any. + if (activePlaybackRef.current) { + activePlaybackRef.current.reject(new Error("Playback stopped")); + activePlaybackRef.current = null; + } + // Reject all pending promises in the main queue while (queueRef.current.length > 0) { const item = queueRef.current.shift()!; @@ -313,5 +326,13 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { stop, isPlaying: () => isPlaying, clearQueue, + warmup: async () => { + if (!audioInitialized) { + await initialize(); + setAudioInitialized(true); + } + // Ensure playback engine isn't suspended after a previous stop. + resumePlayback(); + }, }; } diff --git a/packages/app/src/hooks/use-audio-player.web.ts b/packages/app/src/hooks/use-audio-player.web.ts index 7c833703c..d42ab847d 100644 --- a/packages/app/src/hooks/use-audio-player.web.ts +++ b/packages/app/src/hooks/use-audio-player.web.ts @@ -1,4 +1,4 @@ -import { useMemo } from "react"; +import { useMemo, useRef, useState } from "react"; export interface AudioPlayerOptions { isDetecting?: () => boolean; @@ -6,22 +6,270 @@ export interface AudioPlayerOptions { } /** - * Web shim for the native two-way audio player. - * We currently don't support Voice mode playback on web. + * Web audio player for server-provided audio chunks. + * + * Supports: + * - `audio/pcm` (assumed PCM16 LE, mono, default 24kHz unless `rate=` is present) + * - formats supported by `AudioContext.decodeAudioData` (e.g. mp3) */ -export function useAudioPlayer(_options?: AudioPlayerOptions) { +export function useAudioPlayer(options?: AudioPlayerOptions) { + const [isPlayingState, setIsPlayingState] = useState(false); + + type QueuedAudio = { + audioData: Blob; + resolve: (duration: number) => void; + reject: (error: Error) => void; + }; + + const audioContextRef = useRef(null); + const queueRef = useRef([]); + const suppressedQueueRef = useRef([]); + const isProcessingQueueRef = useRef(false); + const checkIntervalRef = useRef | null>(null); + const activePlaybackRef = useRef<{ + source: AudioBufferSourceNode; + resolve: (duration: number) => void; + reject: (error: Error) => void; + settled: boolean; + } | null>(null); + + const decodeAudioData = async (context: AudioContext, buffer: ArrayBuffer): Promise => { + const maybePromise = context.decodeAudioData(buffer); + if (maybePromise && typeof (maybePromise as Promise).then === "function") { + return maybePromise as Promise; + } + return await new Promise((resolve, reject) => { + context.decodeAudioData(buffer, resolve, reject); + }); + }; + + const parsePcmSampleRate = (mimeType: string): number | null => { + const match = /rate=(\\d+)/i.exec(mimeType); + if (!match) { + return null; + } + const rate = Number(match[1]); + return Number.isFinite(rate) && rate > 0 ? rate : null; + }; + + const pcm16LeToAudioBuffer = (context: AudioContext, bytes: Uint8Array, sampleRate: number): AudioBuffer => { + const sampleCount = Math.floor(bytes.length / 2); + const audioBuffer = context.createBuffer(1, sampleCount, sampleRate); + const channel = audioBuffer.getChannelData(0); + for (let i = 0; i < sampleCount; i++) { + const lo = bytes[i * 2]!; + const hi = bytes[i * 2 + 1]!; + let value = (hi << 8) | lo; + if (value & 0x8000) { + value = value - 0x10000; + } + channel[i] = value / 0x8000; + } + return audioBuffer; + }; + + const ensureContext = async (): Promise => { + if (audioContextRef.current) { + if (audioContextRef.current.state === "suspended") { + try { + await audioContextRef.current.resume(); + } catch { + // Best effort. If this fails due to autoplay policies, a later user gesture + // (or explicit warmup call) can unlock it. + } + } + return audioContextRef.current; + } + const context = new AudioContext(); + if (context.state === "suspended") { + try { + await context.resume(); + } catch { + // See note above. + } + } + audioContextRef.current = context; + return context; + }; + + const shouldSuppressPlayback = (): boolean => { + return Boolean(options?.isDetecting?.()) || Boolean(options?.isSpeaking?.()); + }; + + const startCheckingForClearFlags = (): void => { + if (checkIntervalRef.current !== null) { + return; + } + + checkIntervalRef.current = setInterval(() => { + const isStillBlocked = shouldSuppressPlayback(); + if (!isStillBlocked && suppressedQueueRef.current.length > 0) { + const suppressedItems = [...suppressedQueueRef.current]; + suppressedQueueRef.current = []; + queueRef.current = [...suppressedItems, ...queueRef.current]; + + if (checkIntervalRef.current !== null) { + clearInterval(checkIntervalRef.current); + checkIntervalRef.current = null; + } + + if (!isProcessingQueueRef.current) { + void processQueue(); + } + } else if (!isStillBlocked && suppressedQueueRef.current.length === 0) { + if (checkIntervalRef.current !== null) { + clearInterval(checkIntervalRef.current); + checkIntervalRef.current = null; + } + } + }, 100); + }; + + const playAudio = async (audioData: Blob): Promise => { + const context = await ensureContext(); + const arrayBuffer = await audioData.arrayBuffer(); + + let audioBuffer: AudioBuffer; + const type = (audioData.type || "").toLowerCase(); + + if (type.startsWith("audio/pcm")) { + const sampleRate = parsePcmSampleRate(type) ?? 24000; + audioBuffer = pcm16LeToAudioBuffer(context, new Uint8Array(arrayBuffer), sampleRate); + } else { + audioBuffer = await decodeAudioData(context, arrayBuffer); + } + + const durationSec = audioBuffer.duration; + const source = context.createBufferSource(); + source.buffer = audioBuffer; + source.connect(context.destination); + + return await new Promise((resolve, reject) => { + activePlaybackRef.current = { source, resolve, reject, settled: false }; + setIsPlayingState(true); + + const settleOnce = (fn: () => void) => { + const active = activePlaybackRef.current; + if (!active || active.source !== source || active.settled) { + return; + } + active.settled = true; + activePlaybackRef.current = null; + setIsPlayingState(false); + fn(); + }; + + source.onended = () => { + settleOnce(() => resolve(durationSec)); + }; + + try { + source.start(); + } catch (e) { + settleOnce(() => reject(e instanceof Error ? e : new Error(String(e)))); + } + }); + }; + + const processQueue = async (): Promise => { + if (isProcessingQueueRef.current || queueRef.current.length === 0) { + return; + } + + isProcessingQueueRef.current = true; + + while (queueRef.current.length > 0) { + if (shouldSuppressPlayback()) { + suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current]; + queueRef.current = []; + startCheckingForClearFlags(); + break; + } + + const item = queueRef.current.shift()!; + try { + const duration = await playAudio(item.audioData); + item.resolve(duration); + } catch (error) { + item.reject(error as Error); + } + } + + isProcessingQueueRef.current = false; + }; + + const play = async (audioData: Blob): Promise => { + return await new Promise((resolve, reject) => { + if (shouldSuppressPlayback()) { + suppressedQueueRef.current.push({ audioData, resolve, reject }); + startCheckingForClearFlags(); + return; + } + + queueRef.current.push({ audioData, resolve, reject }); + if (!isProcessingQueueRef.current) { + void processQueue(); + } + }); + }; + + const stop = (): void => { + // Stop currently playing audio (and reject its promise) + if (activePlaybackRef.current) { + const active = activePlaybackRef.current; + activePlaybackRef.current = null; + try { + active.source.onended = null; + active.source.stop(); + } catch { + // ignore + } + if (!active.settled) { + active.settled = true; + active.reject(new Error("Playback stopped")); + } + } + + setIsPlayingState(false); + + while (queueRef.current.length > 0) { + queueRef.current.shift()!.reject(new Error("Playback stopped")); + } + while (suppressedQueueRef.current.length > 0) { + suppressedQueueRef.current.shift()!.reject(new Error("Playback stopped")); + } + + if (checkIntervalRef.current !== null) { + clearInterval(checkIntervalRef.current); + checkIntervalRef.current = null; + } + + isProcessingQueueRef.current = false; + }; + + const clearQueue = (): void => { + while (queueRef.current.length > 0) { + queueRef.current.shift()!.reject(new Error("Queue cleared")); + } + while (suppressedQueueRef.current.length > 0) { + suppressedQueueRef.current.shift()!.reject(new Error("Queue cleared")); + } + if (checkIntervalRef.current !== null) { + clearInterval(checkIntervalRef.current); + checkIntervalRef.current = null; + } + }; + return useMemo( () => ({ - play: async () => { - console.warn("[AudioPlayer] Web playback is disabled"); - return 0; + play, + stop, + isPlaying: () => isPlayingState, + clearQueue, + warmup: async () => { + await ensureContext(); }, - stop: () => { - console.warn("[AudioPlayer] Web playback stop noop"); - }, - isPlaying: () => false, - clearQueue: () => {}, }), - [] + [isPlayingState] ); } diff --git a/packages/app/src/hooks/use-speechmatics-audio.native.ts b/packages/app/src/hooks/use-speechmatics-audio.native.ts index 6bdc9fad6..8c8b9d59e 100644 --- a/packages/app/src/hooks/use-speechmatics-audio.native.ts +++ b/packages/app/src/hooks/use-speechmatics-audio.native.ts @@ -1,5 +1,4 @@ import { useState, useEffect, useRef, useCallback } from "react"; -import { Buffer } from "buffer"; import { initialize, useMicrophonePermissions, @@ -10,6 +9,8 @@ import { type VolumeLevelCallback, } from "@boudra/expo-two-way-audio"; +import { SpeechSegmenter } from "@/voice/speech-segmenter"; + export interface SpeechmaticsAudioConfig { onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void; onSpeechStart?: () => void; @@ -35,31 +36,6 @@ export interface SpeechmaticsAudio { segmentDuration: number; } -function uint8ArrayToBase64(bytes: Uint8Array): string { - // NOTE: This is performance-sensitive during continuous streaming. - // Buffer-backed base64 is significantly faster than manual string building. - try { - return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64"); - } catch { - let binary = ""; - for (let i = 0; i < bytes.length; i++) { - binary += String.fromCharCode(bytes[i]); - } - return btoa(binary); - } -} - -function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array { - const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0); - const result = new Uint8Array(totalLength); - let offset = 0; - for (const arr of arrays) { - result.set(arr, offset); - offset += arr.length; - } - return result; -} - /** * Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio */ @@ -78,49 +54,64 @@ export function useSpeechmaticsAudio( const enableContinuousStreaming = config.enableContinuousStreaming === true; - const audioBufferRef = useRef([]); - const silenceStartRef = useRef(null); - const isSpeakingRef = useRef(false); - const speechDetectionStartRef = useRef(null); - const speechConfirmedRef = useRef(false); - const detectionSilenceStartRef = useRef(null); - const bufferedBytesRef = useRef(0); + const isActiveRef = useRef(isActive); + useEffect(() => { + isActiveRef.current = isActive; + }, [isActive]); - const PCM_SAMPLE_RATE = 16000; - const PCM_CHANNELS = 1; - const PCM_BITS_PER_SAMPLE = 16; - const PCM_BYTES_PER_MS = - (PCM_SAMPLE_RATE * PCM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8)) / 1000; - const MIN_CHUNK_DURATION_MS = 1000; - const MIN_CHUNK_BYTES = Math.round( - PCM_BYTES_PER_MS * MIN_CHUNK_DURATION_MS - ); + const isMutedRef = useRef(isMuted); + useEffect(() => { + isMutedRef.current = isMuted; + }, [isMuted]); - const flushBufferedAudio = useCallback( - (isLast: boolean) => { - if (audioBufferRef.current.length === 0) { - bufferedBytesRef.current = 0; - return; + const callbacksRef = useRef({ + onAudioSegment: config.onAudioSegment, + onSpeechStart: config.onSpeechStart, + onSpeechEnd: config.onSpeechEnd, + }); + useEffect(() => { + callbacksRef.current = { + onAudioSegment: config.onAudioSegment, + onSpeechStart: config.onSpeechStart, + onSpeechEnd: config.onSpeechEnd, + }; + }, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd]); + + const segmenterRef = useRef(null); + if (segmenterRef.current === null) { + segmenterRef.current = new SpeechSegmenter( + { + enableContinuousStreaming, + volumeThreshold: config.volumeThreshold, + silenceDurationMs: config.silenceDuration, + speechConfirmationMs: config.speechConfirmationDuration, + detectionGracePeriodMs: config.detectionGracePeriod, + }, + { + onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment), + onSpeechStart: () => callbacksRef.current.onSpeechStart?.(), + onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(), + onDetectingChange: (next) => setIsDetecting(next), + onSpeakingChange: (next) => setIsSpeaking(next), } + ); + } - const combinedBinary = concatenateUint8Arrays(audioBufferRef.current); - const pcmBase64 = uint8ArrayToBase64(combinedBinary); - - config.onAudioSegment?.({ - audioData: pcmBase64, - isLast, - }); - - audioBufferRef.current = []; - bufferedBytesRef.current = 0; - }, - [config] - ); - - const VOLUME_THRESHOLD = config.volumeThreshold; - const SILENCE_DURATION_MS = config.silenceDuration; - const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration; - const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod; + useEffect(() => { + segmenterRef.current?.updateConfig({ + enableContinuousStreaming, + volumeThreshold: config.volumeThreshold, + silenceDurationMs: config.silenceDuration, + speechConfirmationMs: config.speechConfirmationDuration, + detectionGracePeriodMs: config.detectionGracePeriod, + }); + }, [ + enableContinuousStreaming, + config.volumeThreshold, + config.silenceDuration, + config.speechConfirmationDuration, + config.detectionGracePeriod, + ]); // Update segment duration timer useEffect(() => { @@ -129,7 +120,7 @@ export function useSpeechmaticsAudio( return; } - const startTime = speechDetectionStartRef.current || Date.now(); + const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now(); const interval = setInterval(() => { const elapsed = Date.now() - startTime; setSegmentDuration(elapsed); @@ -143,34 +134,12 @@ export function useSpeechmaticsAudio( "onMicrophoneData", useCallback( (event) => { - if (!isActive || isMuted) return; + if (!isActiveRef.current || isMutedRef.current) return; const pcmData: Uint8Array = event.data; - - if (enableContinuousStreaming) { - audioBufferRef.current.push(pcmData); - bufferedBytesRef.current += pcmData.length; - if (bufferedBytesRef.current >= MIN_CHUNK_BYTES) { - flushBufferedAudio(false); - } - return; - } - - // Buffer the audio chunk if we're detecting or speaking - // Start buffering from first spike to capture beginning of speech - if (speechDetectionStartRef.current !== null || isSpeakingRef.current) { - audioBufferRef.current.push(pcmData); - bufferedBytesRef.current += pcmData.length; - - if ( - speechConfirmedRef.current && - bufferedBytesRef.current >= MIN_CHUNK_BYTES - ) { - flushBufferedAudio(false); - } - } + segmenterRef.current?.pushPcmChunk(pcmData); }, - [enableContinuousStreaming, isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES] + [] ) ); @@ -179,128 +148,15 @@ export function useSpeechmaticsAudio( "onInputVolumeLevelData", useCallback( (event) => { - if (!isActive) return; + if (!isActiveRef.current) return; const volumeLevel: number = event.data; setVolume(volumeLevel); - if (isMuted) return; - if (enableContinuousStreaming) return; - - const speechDetected = volumeLevel > VOLUME_THRESHOLD; - - // console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD); - - if ( - speechDetected && - !isSpeakingRef.current && - !speechConfirmedRef.current - ) { - // Initial speech detection - start tracking - if (speechDetectionStartRef.current === null) { - console.log( - "[SpeechmaticsAudio] Speech started (volume:", - volumeLevel.toFixed(4), - ")" - ); - speechDetectionStartRef.current = Date.now(); - detectionSilenceStartRef.current = null; - audioBufferRef.current = []; - bufferedBytesRef.current = 0; - setIsDetecting(true); - } else { - // Volume is back above threshold - reset grace period - detectionSilenceStartRef.current = null; - - // Check if speech has been sustained long enough - const speechDuration = Date.now() - speechDetectionStartRef.current; - if (speechDuration >= SPEECH_CONFIRMATION_MS) { - // Speech CONFIRMED - console.log( - "[SpeechmaticsAudio] Speech confirmed after", - speechDuration, - "ms" - ); - isSpeakingRef.current = true; - speechConfirmedRef.current = true; - silenceStartRef.current = null; - setIsDetecting(false); - setIsSpeaking(true); - config.onSpeechStart?.(); - } - } - } else if ( - speechDetected && - isSpeakingRef.current && - speechConfirmedRef.current - ) { - // Continuing confirmed speech - silenceStartRef.current = null; - } else if ( - !speechDetected && - !speechConfirmedRef.current && - speechDetectionStartRef.current !== null - ) { - // Volume dropped during detection phase - apply grace period - if (detectionSilenceStartRef.current === null) { - detectionSilenceStartRef.current = Date.now(); - } else { - const graceDuration = Date.now() - detectionSilenceStartRef.current; - if (graceDuration >= DETECTION_GRACE_PERIOD_MS) { - // Grace period expired - cancel detection - console.log( - "[SpeechmaticsAudio] Speech detection cancelled after", - graceDuration, - "ms grace period" - ); - speechDetectionStartRef.current = null; - detectionSilenceStartRef.current = null; - audioBufferRef.current = []; - bufferedBytesRef.current = 0; - setIsDetecting(false); - } - } - } else if ( - !speechDetected && - isSpeakingRef.current && - speechConfirmedRef.current - ) { - // Potential speech END - if (silenceStartRef.current === null) { - silenceStartRef.current = Date.now(); - } else { - const silenceDuration = Date.now() - silenceStartRef.current; - if (silenceDuration >= SILENCE_DURATION_MS) { - // Speech END confirmed - console.log( - "[SpeechmaticsAudio] Speech ended after", - silenceDuration, - "ms silence" - ); - isSpeakingRef.current = false; - speechConfirmedRef.current = false; - speechDetectionStartRef.current = null; - silenceStartRef.current = null; - setIsSpeaking(false); - config.onSpeechEnd?.(); - - // Send buffered audio segment - flushBufferedAudio(true); - } - } - } + if (isMutedRef.current) return; + segmenterRef.current?.pushVolumeLevel(volumeLevel, Date.now()); }, - [ - enableContinuousStreaming, - isActive, - isMuted, - VOLUME_THRESHOLD, - SILENCE_DURATION_MS, - SPEECH_CONFIRMATION_MS, - DETECTION_GRACE_PERIOD_MS, - config, - flushBufferedAudio, - ] + [] ) ); @@ -363,9 +219,7 @@ export function useSpeechmaticsAudio( toggleRecording(false); } - if (enableContinuousStreaming) { - flushBufferedAudio(true); - } + segmenterRef.current?.stop(Date.now()); // Tear down audio session if (audioInitialized) { @@ -375,13 +229,7 @@ export function useSpeechmaticsAudio( } // Reset state - audioBufferRef.current = []; - bufferedBytesRef.current = 0; - isSpeakingRef.current = false; - speechConfirmedRef.current = false; - speechDetectionStartRef.current = null; - detectionSilenceStartRef.current = null; - silenceStartRef.current = null; + segmenterRef.current?.reset(); setIsActive(false); setIsSpeaking(false); setIsDetecting(false); @@ -398,13 +246,7 @@ export function useSpeechmaticsAudio( if (newMuted) { // Clear any ongoing speech detection/speaking state - audioBufferRef.current = []; - bufferedBytesRef.current = 0; - isSpeakingRef.current = false; - speechConfirmedRef.current = false; - speechDetectionStartRef.current = null; - detectionSilenceStartRef.current = null; - silenceStartRef.current = null; + segmenterRef.current?.reset(); setIsSpeaking(false); setIsDetecting(false); } diff --git a/packages/app/src/hooks/use-speechmatics-audio.web.ts b/packages/app/src/hooks/use-speechmatics-audio.web.ts index 320e83b66..6bf605e06 100644 --- a/packages/app/src/hooks/use-speechmatics-audio.web.ts +++ b/packages/app/src/hooks/use-speechmatics-audio.web.ts @@ -1,15 +1,18 @@ -import { useCallback, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +import { SpeechSegmenter } from "@/voice/speech-segmenter"; export interface SpeechmaticsAudioConfig { onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void; onSpeechStart?: () => void; onSpeechEnd?: () => void; onError?: (error: Error) => void; + /** When true, stream microphone PCM continuously without VAD gating. */ enableContinuousStreaming?: boolean; - volumeThreshold: number; - silenceDuration: number; - speechConfirmationDuration: number; - detectionGracePeriod: number; + volumeThreshold: number; // 0-1 + silenceDuration: number; // ms of silence before ending segment + speechConfirmationDuration: number; // ms of sustained speech before confirming + detectionGracePeriod: number; // ms grace period for volume dips during detection } export interface SpeechmaticsAudio { @@ -24,41 +27,324 @@ export interface SpeechmaticsAudio { segmentDuration: number; } -/** - * Web shim for the Speechmatics two-way audio hook. - * Real-time microphone capture is currently unsupported on web. - */ -export function useSpeechmaticsAudio( - _config: SpeechmaticsAudioConfig -): SpeechmaticsAudio { +const getAudioContextCtor = (): (typeof AudioContext) | null => { + if (typeof window === "undefined") { + return null; + } + const ctor = + (window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext || + (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; + return ctor ?? null; +}; + +const floatToInt16 = (sample: number): number => { + const clamped = Math.max(-1, Math.min(1, sample)); + return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff); +}; + +const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => { + if (input.length === 0) { + return new Int16Array(0); + } + if (inputRate === outputRate) { + const out = new Int16Array(input.length); + for (let i = 0; i < input.length; i++) { + out[i] = floatToInt16(input[i]); + } + return out; + } + + const ratio = inputRate / outputRate; + const outputLength = Math.max(1, Math.round(input.length / ratio)); + const out = new Int16Array(outputLength); + for (let i = 0; i < outputLength; i++) { + const sourceIndex = i * ratio; + const i0 = Math.floor(sourceIndex); + const i1 = Math.min(input.length - 1, i0 + 1); + const frac = sourceIndex - i0; + const sample = input[i0] * (1 - frac) + input[i1] * frac; + out[i] = floatToInt16(sample); + } + return out; +}; + +export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): SpeechmaticsAudio { const [isActive, setIsActive] = useState(false); + const [isSpeaking, setIsSpeaking] = useState(false); + const [isDetecting, setIsDetecting] = useState(false); + const [volume, setVolume] = useState(0); const [isMuted, setIsMuted] = useState(false); + const [segmentDuration, setSegmentDuration] = useState(0); + + const enableContinuousStreaming = config.enableContinuousStreaming === true; + + const callbacksRef = useRef({ + onAudioSegment: config.onAudioSegment, + onSpeechStart: config.onSpeechStart, + onSpeechEnd: config.onSpeechEnd, + onError: config.onError, + }); + + useEffect(() => { + callbacksRef.current = { + onAudioSegment: config.onAudioSegment, + onSpeechStart: config.onSpeechStart, + onSpeechEnd: config.onSpeechEnd, + onError: config.onError, + }; + }, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd, config.onError]); + + const segmenterRef = useRef(null); + if (segmenterRef.current === null) { + segmenterRef.current = new SpeechSegmenter( + { + enableContinuousStreaming, + volumeThreshold: config.volumeThreshold, + silenceDurationMs: config.silenceDuration, + speechConfirmationMs: config.speechConfirmationDuration, + detectionGracePeriodMs: config.detectionGracePeriod, + }, + { + onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment), + onSpeechStart: () => callbacksRef.current.onSpeechStart?.(), + onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(), + onDetectingChange: (next) => setIsDetecting(next), + onSpeakingChange: (next) => setIsSpeaking(next), + } + ); + } + + useEffect(() => { + segmenterRef.current?.updateConfig({ + enableContinuousStreaming, + volumeThreshold: config.volumeThreshold, + silenceDurationMs: config.silenceDuration, + speechConfirmationMs: config.speechConfirmationDuration, + detectionGracePeriodMs: config.detectionGracePeriod, + }); + }, [ + enableContinuousStreaming, + config.volumeThreshold, + config.silenceDuration, + config.speechConfirmationDuration, + config.detectionGracePeriod, + ]); + + const refs = useRef<{ + started: boolean; + stream: MediaStream | null; + context: AudioContext | null; + source: MediaStreamAudioSourceNode | null; + processor: ScriptProcessorNode | null; + gain: GainNode | null; + }>({ + started: false, + stream: null, + context: null, + source: null, + processor: null, + gain: null, + }); + + const isMutedRef = useRef(isMuted); + useEffect(() => { + isMutedRef.current = isMuted; + }, [isMuted]); + + // Update segment duration timer + useEffect(() => { + if (!isDetecting && !isSpeaking) { + setSegmentDuration(0); + return; + } + + const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now(); + const interval = setInterval(() => { + setSegmentDuration(Date.now() - startTime); + }, 100); + + return () => clearInterval(interval); + }, [isDetecting, isSpeaking]); + + const stopInternal = useCallback(async () => { + refs.current.started = false; + + try { + refs.current.processor?.disconnect(); + refs.current.source?.disconnect(); + refs.current.gain?.disconnect(); + } catch { + // best-effort teardown + } + + if (refs.current.stream) { + for (const track of refs.current.stream.getTracks()) { + try { + track.stop(); + } catch { + // ignore + } + } + } + + const context = refs.current.context; + if (context && context.state !== "closed") { + try { + await context.close(); + } catch { + // ignore + } + } + + refs.current.stream = null; + refs.current.context = null; + refs.current.source = null; + refs.current.processor = null; + refs.current.gain = null; + + segmenterRef.current?.stop(Date.now()); + setIsActive(false); + setIsSpeaking(false); + setIsDetecting(false); + setVolume(0); + setIsMuted(false); + }, []); const start = useCallback(async () => { - console.warn("[SpeechmaticsAudio] Web microphone capture disabled"); - setIsActive(true); - }, []); + if (refs.current.started) { + return; + } + + const missingNavigator = + typeof navigator === "undefined" || + !navigator.mediaDevices || + typeof navigator.mediaDevices.getUserMedia !== "function"; + + const secureContext = + typeof window !== "undefined" && typeof window.isSecureContext === "boolean" + ? window.isSecureContext + : true; + const currentOrigin = typeof window !== "undefined" && window.location ? window.location.origin : "unknown"; + + try { + if (missingNavigator) { + throw new Error("Microphone capture is not supported in this environment"); + } + if (!secureContext) { + throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`); + } + + const AudioContextCtor = getAudioContextCtor(); + if (!AudioContextCtor) { + throw new Error("AudioContext unavailable"); + } + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + noiseSuppression: true, + echoCancellation: true, + autoGainControl: true, + }, + }); + + const context = new AudioContextCtor(); + if (context.state === "suspended") { + await context.resume(); + } + + const source = context.createMediaStreamSource(stream); + const processor = context.createScriptProcessor(4096, 1, 1); + const gain = context.createGain(); + gain.gain.value = 0; + + refs.current = { + started: true, + stream, + context, + source, + processor, + gain, + }; + + processor.onaudioprocess = (event) => { + if (!refs.current.started) { + return; + } + + const input = event.inputBuffer.getChannelData(0); + let sumSquares = 0; + for (let i = 0; i < input.length; i++) { + const sample = input[i]; + sumSquares += sample * sample; + } + const rms = Math.sqrt(sumSquares / Math.max(1, input.length)); + const normalized = Math.min(1, Math.max(0, rms * 2)); + setVolume(normalized); + + if (isMutedRef.current) { + return; + } + + const nowMs = Date.now(); + segmenterRef.current?.pushVolumeLevel(normalized, nowMs); + + const pcm16 = resampleToPcm16(input, context.sampleRate, 16000); + const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength); + segmenterRef.current?.pushPcmChunk(pcmBytes); + }; + + source.connect(processor); + processor.connect(gain); + gain.connect(context.destination); + + setIsActive(true); + return; + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + callbacksRef.current.onError?.(err); + await stopInternal(); + throw err; + } + }, [stopInternal]); const stop = useCallback(async () => { - setIsActive(false); - }, []); + await stopInternal(); + }, [stopInternal]); const toggleMute = useCallback(() => { - setIsMuted((prev) => !prev); + setIsMuted((prev) => { + const nextMuted = !prev; + if (nextMuted) { + segmenterRef.current?.reset(); + setIsSpeaking(false); + setIsDetecting(false); + } + return nextMuted; + }); }, []); + useEffect(() => { + return () => { + if (refs.current.started) { + void stopInternal(); + } + }; + }, [stopInternal]); + return useMemo( () => ({ start, stop, toggleMute, isActive, - isSpeaking: false, - isDetecting: false, + isSpeaking, + isDetecting, isMuted, - volume: 0, - segmentDuration: 0, + volume, + segmentDuration, }), - [start, stop, toggleMute, isActive, isMuted] + [start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration] ); } + diff --git a/packages/app/src/voice/speech-segmenter.test.ts b/packages/app/src/voice/speech-segmenter.test.ts new file mode 100644 index 000000000..c9b0278a5 --- /dev/null +++ b/packages/app/src/voice/speech-segmenter.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it, vi } from "vitest"; + +import { SpeechSegmenter } from "./speech-segmenter"; + +const mkPcmBytes = (length: number): Uint8Array => { + const bytes = new Uint8Array(length); + for (let i = 0; i < length; i++) bytes[i] = i % 255; + return bytes; +}; + +describe("SpeechSegmenter", () => { + it("streams continuously and flushes at stop", () => { + const onAudioSegment = vi.fn(); + const segmenter = new SpeechSegmenter( + { + enableContinuousStreaming: true, + volumeThreshold: 0.3, + silenceDurationMs: 2000, + speechConfirmationMs: 300, + detectionGracePeriodMs: 200, + minChunkDurationMs: 100, // ensure we only flush on stop + pcmSampleRate: 1000, + }, + { onAudioSegment } + ); + + segmenter.pushPcmChunk(mkPcmBytes(50)); + segmenter.pushPcmChunk(mkPcmBytes(50)); + expect(onAudioSegment).not.toHaveBeenCalled(); + + segmenter.stop(Date.now()); + + expect(onAudioSegment).toHaveBeenCalledTimes(1); + const lastCall = onAudioSegment.mock.calls.at(-1)![0]; + expect(lastCall.isLast).toBe(true); + expect(lastCall.audioData.length).toBeGreaterThan(0); + }); + + it("detects speech, calls onSpeechStart, and flushes on speech end", () => { + const onAudioSegment = vi.fn(); + const onSpeechStart = vi.fn(); + const onSpeechEnd = vi.fn(); + const detectingChanges: boolean[] = []; + const speakingChanges: boolean[] = []; + + const segmenter = new SpeechSegmenter( + { + enableContinuousStreaming: false, + volumeThreshold: 0.3, + silenceDurationMs: 2000, + speechConfirmationMs: 300, + detectionGracePeriodMs: 200, + minChunkDurationMs: 100, + pcmSampleRate: 1000, + }, + { + onAudioSegment, + onSpeechStart, + onSpeechEnd, + onDetectingChange: (v) => detectingChanges.push(v), + onSpeakingChange: (v) => speakingChanges.push(v), + } + ); + + const t0 = 10_000; + + // Start detection. + segmenter.pushVolumeLevel(0.5, t0); + segmenter.pushPcmChunk(mkPcmBytes(50)); + expect(detectingChanges).toEqual([true]); + expect(onSpeechStart).not.toHaveBeenCalled(); + + // Confirm speech. + segmenter.pushVolumeLevel(0.5, t0 + 300); + segmenter.pushPcmChunk(mkPcmBytes(50)); + expect(onSpeechStart).toHaveBeenCalledTimes(1); + expect(speakingChanges).toEqual([true]); + + // End speech after silence. + segmenter.pushVolumeLevel(0.0, t0 + 301); + segmenter.pushPcmChunk(mkPcmBytes(50)); + segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000); + + expect(onSpeechEnd).toHaveBeenCalledTimes(1); + expect(speakingChanges).toEqual([true, false]); + expect(onAudioSegment).toHaveBeenCalled(); + const lastCall = onAudioSegment.mock.calls.at(-1)![0]; + expect(lastCall.isLast).toBe(true); + expect(lastCall.audioData.length).toBeGreaterThan(0); + }); +}); diff --git a/packages/app/src/voice/speech-segmenter.ts b/packages/app/src/voice/speech-segmenter.ts new file mode 100644 index 000000000..08b7e095f --- /dev/null +++ b/packages/app/src/voice/speech-segmenter.ts @@ -0,0 +1,285 @@ +import { Buffer } from "buffer"; + +export type SpeechSegment = { audioData: string; isLast: boolean }; + +export type SpeechSegmenterConfig = { + enableContinuousStreaming: boolean; + volumeThreshold: number; // 0..1 + silenceDurationMs: number; + speechConfirmationMs: number; + detectionGracePeriodMs: number; + minChunkDurationMs?: number; + pcmSampleRate?: number; + pcmChannels?: number; + pcmBitsPerSample?: number; +}; + +export type SpeechSegmenterCallbacks = { + onAudioSegment?: (segment: SpeechSegment) => void; + onSpeechStart?: () => void; + onSpeechEnd?: () => void; + onDetectingChange?: (isDetecting: boolean) => void; + onSpeakingChange?: (isSpeaking: boolean) => void; +}; + +function uint8ArrayToBase64(bytes: Uint8Array): string { + // NOTE: This is performance-sensitive during continuous streaming. + // Buffer-backed base64 is significantly faster than manual string building. + try { + return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64"); + } catch { + let binary = ""; + for (let i = 0; i < bytes.length; i++) { + binary += String.fromCharCode(bytes[i]); + } + // eslint-disable-next-line no-restricted-globals + return btoa(binary); + } +} + +function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array { + const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const arr of arrays) { + result.set(arr, offset); + offset += arr.length; + } + return result; +} + +export class SpeechSegmenter { + private config: Required> & + Required>; + private callbacks: SpeechSegmenterCallbacks; + + private audioBuffer: Uint8Array[] = []; + private bufferedBytes = 0; + + private speechDetectionStartMs: number | null = null; + private detectionSilenceStartMs: number | null = null; + private silenceStartMs: number | null = null; + private isSpeaking = false; + private isDetecting = false; + private speechConfirmed = false; + + private pcmBytesPerMs: number; + private minChunkBytes: number; + + constructor(config: SpeechSegmenterConfig, callbacks: SpeechSegmenterCallbacks = {}) { + this.config = { + ...config, + minChunkDurationMs: config.minChunkDurationMs ?? 1000, + pcmSampleRate: config.pcmSampleRate ?? 16000, + pcmChannels: config.pcmChannels ?? 1, + pcmBitsPerSample: config.pcmBitsPerSample ?? 16, + }; + this.callbacks = callbacks; + this.pcmBytesPerMs = 0; + this.minChunkBytes = 0; + this.recomputeDerived(); + } + + updateConfig(next: SpeechSegmenterConfig): void { + this.config = { + ...next, + minChunkDurationMs: next.minChunkDurationMs ?? this.config.minChunkDurationMs, + pcmSampleRate: next.pcmSampleRate ?? this.config.pcmSampleRate, + pcmChannels: next.pcmChannels ?? this.config.pcmChannels, + pcmBitsPerSample: next.pcmBitsPerSample ?? this.config.pcmBitsPerSample, + }; + this.recomputeDerived(); + } + + setCallbacks(callbacks: SpeechSegmenterCallbacks): void { + this.callbacks = callbacks; + } + + getSpeechDetectionStartMs(): number | null { + return this.speechDetectionStartMs; + } + + getIsSpeaking(): boolean { + return this.isSpeaking; + } + + getIsDetecting(): boolean { + return this.isDetecting; + } + + reset(): void { + this.audioBuffer = []; + this.bufferedBytes = 0; + this.speechDetectionStartMs = null; + this.detectionSilenceStartMs = null; + this.silenceStartMs = null; + this.isSpeaking = false; + this.isDetecting = false; + this.speechConfirmed = false; + } + + flush(isLast: boolean): void { + if (this.audioBuffer.length === 0) { + this.bufferedBytes = 0; + return; + } + + const combinedBinary = concatenateUint8Arrays(this.audioBuffer); + const pcmBase64 = uint8ArrayToBase64(combinedBinary); + this.callbacks.onAudioSegment?.({ audioData: pcmBase64, isLast }); + + this.audioBuffer = []; + this.bufferedBytes = 0; + } + + /** + * Called when the input stream emits a PCM16 chunk (Uint8Array of bytes). + * In non-continuous mode, chunk buffering is gated by the VAD state. + */ + pushPcmChunk(chunk: Uint8Array): void { + if (chunk.length === 0) { + return; + } + + if (this.config.enableContinuousStreaming) { + this.audioBuffer.push(chunk); + this.bufferedBytes += chunk.length; + if (this.bufferedBytes >= this.minChunkBytes) { + this.flush(false); + } + return; + } + + // Buffer if we're detecting or speaking. When speech is confirmed, we also + // stream partial chunks to reduce latency. + if (this.speechDetectionStartMs !== null || this.isSpeaking) { + this.audioBuffer.push(chunk); + this.bufferedBytes += chunk.length; + + if (this.speechConfirmed && this.bufferedBytes >= this.minChunkBytes) { + this.flush(false); + } + } + } + + /** + * Called with a normalized volume level (0..1). Drives VAD transitions. + */ + pushVolumeLevel(volume: number, nowMs: number): void { + if (this.config.enableContinuousStreaming) { + return; + } + + const speechDetected = volume > this.config.volumeThreshold; + + if (speechDetected && !this.isSpeaking && !this.speechConfirmed) { + // Initial detection phase. + if (this.speechDetectionStartMs === null) { + this.speechDetectionStartMs = nowMs; + this.detectionSilenceStartMs = null; + this.audioBuffer = []; + this.bufferedBytes = 0; + this.setDetecting(true); + return; + } + + // Volume is back above threshold - reset grace period. + this.detectionSilenceStartMs = null; + + const speechDuration = nowMs - this.speechDetectionStartMs; + if (speechDuration >= this.config.speechConfirmationMs) { + this.speechConfirmed = true; + this.silenceStartMs = null; + this.setDetecting(false); + this.setSpeaking(true); + this.callbacks.onSpeechStart?.(); + } + return; + } + + if (speechDetected && this.isSpeaking && this.speechConfirmed) { + // Continuing confirmed speech. + this.silenceStartMs = null; + return; + } + + if (!speechDetected && !this.speechConfirmed && this.speechDetectionStartMs !== null) { + // Volume dropped during detection phase - apply grace period. + if (this.detectionSilenceStartMs === null) { + this.detectionSilenceStartMs = nowMs; + return; + } + + const graceDuration = nowMs - this.detectionSilenceStartMs; + if (graceDuration >= this.config.detectionGracePeriodMs) { + // Cancel detection. + this.speechDetectionStartMs = null; + this.detectionSilenceStartMs = null; + this.audioBuffer = []; + this.bufferedBytes = 0; + this.setDetecting(false); + } + return; + } + + if (!speechDetected && this.isSpeaking && this.speechConfirmed) { + // Potential speech END. + if (this.silenceStartMs === null) { + this.silenceStartMs = nowMs; + return; + } + + const silenceDuration = nowMs - this.silenceStartMs; + if (silenceDuration >= this.config.silenceDurationMs) { + // Speech END confirmed. + this.speechConfirmed = false; + this.speechDetectionStartMs = null; + this.silenceStartMs = null; + this.setSpeaking(false); + this.callbacks.onSpeechEnd?.(); + if (this.audioBuffer.length === 0) { + // Important: downstream expects an explicit segment boundary marker. + this.callbacks.onAudioSegment?.({ audioData: "", isLast: true }); + } else { + this.flush(true); + } + } + } + } + + stop(nowMs: number): void { + void nowMs; + if (this.config.enableContinuousStreaming) { + this.flush(true); + } + // Ensure observers see state clear transitions. + this.setDetecting(false); + this.setSpeaking(false); + this.reset(); + } + + private setDetecting(next: boolean): void { + if (this.isDetecting === next) { + return; + } + this.isDetecting = next; + this.callbacks.onDetectingChange?.(next); + } + + private setSpeaking(next: boolean): void { + if (this.isSpeaking === next) { + return; + } + this.isSpeaking = next; + this.callbacks.onSpeakingChange?.(next); + } + + private recomputeDerived(): void { + this.pcmBytesPerMs = + (this.config.pcmSampleRate * + this.config.pcmChannels * + (this.config.pcmBitsPerSample / 8)) / + 1000; + this.minChunkBytes = Math.round(this.pcmBytesPerMs * this.config.minChunkDurationMs); + } +}