jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Update files

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-11 11:04:49 +07:00
parent 002c09681d
commit 0d60588431
23 changed files with 1264 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/components/volume-meter.tsx
+6 -7
View File
@@ -24,7 +24,6 @@ interface VolumeMeterProps {
export function VolumeMeter({
volume,
isMuted = false,
isDetecting = false,
isSpeaking = false,
orientation = "vertical",
variant = "default",
@@ -146,9 +145,9 @@ export function VolumeMeter({
// Create animated styles unconditionally at top level
const line1Style = useAnimatedStyle(() => {
const isActive = isDetecting || isSpeaking;
const isActive = isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted ? 0 : volume * 0.3;
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
return {
height: line1Height.value * (isMuted || volume > 0.001 ? 1 : line1Pulse.value),
opacity: baseOpacity + volumeBoost,
@@ -156,9 +155,9 @@ export function VolumeMeter({
});
const line2Style = useAnimatedStyle(() => {
const isActive = isDetecting || isSpeaking;
const isActive = isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted ? 0 : volume * 0.3;
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
return {
height: line2Height.value * (isMuted || volume > 0.001 ? 1 : line2Pulse.value),
opacity: baseOpacity + volumeBoost,
@@ -166,9 +165,9 @@ export function VolumeMeter({
});
const line3Style = useAnimatedStyle(() => {
const isActive = isDetecting || isSpeaking;
const isActive = isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted ? 0 : volume * 0.3;
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
return {
height: line3Height.value * (isMuted || volume > 0.001 ? 1 : line3Pulse.value),
opacity: baseOpacity + volumeBoost,
packages/app/src/contexts/voice-context.tsx
+51 -2
View File
@@ -63,10 +63,15 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
const lastVoiceModeSyncedClientRef = useRef<SessionState["client"] | null>(null);
const voiceTransportReadyRef = useRef(false);
const voiceResyncInFlightRef = useRef(false);
const silenceGraceStartMsRef = useRef<number | null>(null);
const vadStateRef = useRef<{ isDetecting: boolean; isSpeaking: boolean }>({
isDetecting: false,
isSpeaking: false,
});
const realtimeAudio = useSpeechmaticsAudio({
onSpeechStart: () => {
console.log("[Voice] Speech detected");
console.log("[Voice] Segment started (speech confirmed)");
// Stop audio playback if playing
const session = realtimeSessionRef.current;
const sessionAudioPlayer = session?.audioPlayer ?? null;
@@ -93,7 +98,16 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
}
},
onSpeechEnd: () => {
console.log("[Voice] Speech ended");
const silenceMs =
silenceGraceStartMsRef.current === null
? null
: Date.now() - silenceGraceStartMsRef.current;
if (silenceMs === null) {
console.log("[Voice] Segment finalized");
} else {
console.log("[Voice] Segment finalized", { silenceMs });
}
silenceGraceStartMsRef.current = null;
},
onAudioSegment: ({ audioData, isLast }) => {
if (!voiceTransportReadyRef.current) {
@@ -134,6 +148,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
}
},
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
confirmedDropGracePeriod: REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
silenceDuration: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
speechConfirmationDuration: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
detectionGracePeriod: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
@@ -143,6 +158,40 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
realtimeSessionRef.current = activeSession;
}, [activeSession]);
useEffect(() => {
const next = {
isDetecting: realtimeAudio.isDetecting,
isSpeaking: realtimeAudio.isSpeaking,
};
const prev = vadStateRef.current;
if (!isVoiceMode) {
silenceGraceStartMsRef.current = null;
vadStateRef.current = next;
return;
}
// Confirmed speech dropped below threshold: grace window starts.
if (prev.isSpeaking && !next.isSpeaking && next.isDetecting) {
silenceGraceStartMsRef.current = Date.now();
console.log("[Voice] Grace started (speech dropped below threshold)");
}
// User resumed speaking before grace timeout elapsed.
if (!prev.isSpeaking && next.isSpeaking && silenceGraceStartMsRef.current !== null) {
const resumedAfterMs = Date.now() - silenceGraceStartMsRef.current;
console.log("[Voice] Speech resumed during grace", { resumedAfterMs });
silenceGraceStartMsRef.current = null;
}
// Fully idle (neither detecting nor speaking).
if (!next.isDetecting && !next.isSpeaking) {
silenceGraceStartMsRef.current = null;
}
vadStateRef.current = next;
}, [isVoiceMode, realtimeAudio.isDetecting, realtimeAudio.isSpeaking]);
useEffect(() => {
const connected = activeSession?.connection.isConnected ?? false;
const client = activeSession?.client ?? null;
packages/app/src/hooks/use-speechmatics-audio.native.ts
+5
View File
@@ -19,6 +19,8 @@ export interface SpeechmaticsAudioConfig {
/** When true, stream microphone PCM continuously without VAD gating. */
enableContinuousStreaming?: boolean;
volumeThreshold: number; // Volume threshold for speech detection (0-1)
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
confirmedDropGracePeriod?: number;
silenceDuration: number; // ms of silence before ending segment
speechConfirmationDuration: number; // ms of sustained speech before confirming
detectionGracePeriod: number; // ms grace period for volume dips during detection
@@ -83,6 +85,7 @@ export function useSpeechmaticsAudio(
{
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
@@ -101,6 +104,7 @@ export function useSpeechmaticsAudio(
segmenterRef.current?.updateConfig({
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
@@ -108,6 +112,7 @@ export function useSpeechmaticsAudio(
}, [
enableContinuousStreaming,
config.volumeThreshold,
config.confirmedDropGracePeriod,
config.silenceDuration,
config.speechConfirmationDuration,
config.detectionGracePeriod,
packages/app/src/hooks/use-speechmatics-audio.web.ts
+44 -1
View File
@@ -10,6 +10,8 @@ export interface SpeechmaticsAudioConfig {
/** When true, stream microphone PCM continuously without VAD gating. */
enableContinuousStreaming?: boolean;
volumeThreshold: number; // 0-1
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
confirmedDropGracePeriod?: number;
silenceDuration: number; // ms of silence before ending segment
speechConfirmationDuration: number; // ms of sustained speech before confirming
detectionGracePeriod: number; // ms grace period for volume dips during detection
@@ -100,6 +102,7 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
{
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
@@ -118,6 +121,7 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
segmenterRef.current?.updateConfig({
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
@@ -125,6 +129,7 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
}, [
enableContinuousStreaming,
config.volumeThreshold,
config.confirmedDropGracePeriod,
config.silenceDuration,
config.speechConfirmationDuration,
config.detectionGracePeriod,
@@ -151,6 +156,21 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
isMutedRef.current = isMuted;
}, [isMuted]);
const vadConfigRef = useRef({
volumeThreshold: config.volumeThreshold,
});
useEffect(() => {
vadConfigRef.current = {
volumeThreshold: config.volumeThreshold,
};
}, [config.volumeThreshold]);
const vadLogRef = useRef({
lastLogMs: 0,
lastSpeaking: false,
lastDetecting: false,
});
// Update segment duration timer
useEffect(() => {
if (!isDetecting && !isSpeaking) {
@@ -288,6 +308,30 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
const nowMs = Date.now();
segmenterRef.current?.pushVolumeLevel(normalized, nowMs);
const segmenter = segmenterRef.current;
if (segmenter) {
const speakingNow = segmenter.getIsSpeaking();
const detectingNow = segmenter.getIsDetecting();
const shouldLog =
nowMs - vadLogRef.current.lastLogMs >= 150 ||
speakingNow !== vadLogRef.current.lastSpeaking ||
detectingNow !== vadLogRef.current.lastDetecting;
if (shouldLog) {
const threshold = vadConfigRef.current.volumeThreshold;
const releaseThreshold = segmenter.getVolumeReleaseThreshold();
console.log("[Voice][Web][VAD] level", {
volume: Number(normalized.toFixed(3)),
threshold: Number(threshold.toFixed(3)),
releaseThreshold: Number(releaseThreshold.toFixed(3)),
confirmedDropGraceMs: config.confirmedDropGracePeriod ?? 250,
isSpeaking: speakingNow,
isDetecting: detectingNow,
});
vadLogRef.current.lastLogMs = nowMs;
vadLogRef.current.lastSpeaking = speakingNow;
vadLogRef.current.lastDetecting = detectingNow;
}
}
const pcm16 = resampleToPcm16(input, context.sampleRate, 16000);
const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength);
@@ -347,4 +391,3 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
[start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration]
);
}
packages/app/src/screens/agent/agent-ready-screen.tsx
+10 -2
View File
@@ -553,7 +553,10 @@ function AgentScreenContent({
if (!isConnected) {
return;
}
if (!needsAuthoritativeSync) {
// On native clients, daemon stream forwarding is focused-agent only, so switching
// agents can leave timeline gaps unless we explicitly request a snapshot.
const shouldSyncOnEntry = needsAuthoritativeSync || Platform.OS !== "web";
if (!shouldSyncOnEntry) {
return;
}
@@ -563,7 +566,12 @@ function AgentScreenContent({
error,
});
});
}, [resolvedAgentId, ensureAgentIsInitialized, isConnected, needsAuthoritativeSync]);
}, [
resolvedAgentId,
ensureAgentIsInitialized,
isConnected,
needsAuthoritativeSync,
]);
useEffect(() => {
// Clear stale resolution state when route target changes.
packages/app/src/screens/agent/draft-agent-screen.tsx
+5 -2
View File
@@ -400,6 +400,7 @@ export function DraftAgentScreen({
const checkoutQueryError =
checkoutStatusQuery.error instanceof Error ? checkoutStatusQuery.error.message : null;
const checkoutPayloadError = checkout?.error ? checkout.error.message : null;
const isGitDirectory = checkoutStatusQuery.isSuccess && checkout?.isGit === true;
const isNonGitDirectory =
Boolean(trimmedWorkingDir) &&
@@ -430,7 +431,7 @@ export function DraftAgentScreen({
const isCreateWorktree = worktreeMode === "create";
const isAttachWorktree = worktreeMode === "attach";
const worktreeListRoot = checkout?.repoRoot ?? trimmedWorkingDir;
const worktreeListRoot = checkout?.isGit ? checkout.repoRoot : "";
const worktreeListQuery = useQuery({
queryKey: ["paseoWorktreeList", selectedServerId, worktreeListRoot],
queryFn: async () => {
@@ -448,7 +449,8 @@ export function DraftAgentScreen({
return payload.worktrees ?? [];
},
enabled:
Boolean(worktreeListRoot || trimmedWorkingDir) &&
isGitDirectory &&
Boolean(worktreeListRoot) &&
!repoAvailabilityError &&
Boolean(sessionClient) &&
isConnected &&
@@ -551,6 +553,7 @@ export function DraftAgentScreen({
},
enabled:
isCreateWorktree &&
isGitDirectory &&
!isNonGitDirectory &&
Boolean(baseBranch) &&
Boolean(trimmedWorkingDir) &&
packages/app/src/voice/realtime-voice-config.ts
+8 -3
View File
@@ -1,7 +1,12 @@
export const REALTIME_VOICE_VAD_CONFIG = {
volumeThreshold: 0.18,
silenceDurationMs: 2200,
// Headset mics often report lower input volume than built-in mics.
// Keep threshold low enough to avoid requiring loud speech.
volumeThreshold: 0.12,
// Short staggered grace before we visually/VAD-deactivate speaking.
// Prevents fade-out between normal intra-word pauses.
confirmedDropGracePeriodMs: 1000,
// Keep turns open longer so brief "thinking pauses" don't end speech too early.
silenceDurationMs: 2000,
speechConfirmationMs: 120,
detectionGracePeriodMs: 700,
} as const;
packages/app/src/voice/speech-segmenter.test.ts
+148
View File
@@ -68,20 +68,24 @@ describe("SpeechSegmenter", () => {
segmenter.pushVolumeLevel(0.5, t0);
segmenter.pushPcmChunk(mkPcmBytes(50));
expect(detectingChanges).toEqual([true]);
expect(speakingChanges).toEqual([true]);
expect(onSpeechStart).not.toHaveBeenCalled();
// Confirm speech.
segmenter.pushVolumeLevel(0.5, t0 + 300);
segmenter.pushPcmChunk(mkPcmBytes(50));
expect(onSpeechStart).toHaveBeenCalledTimes(1);
expect(detectingChanges).toEqual([true, false]);
expect(speakingChanges).toEqual([true]);
// End speech after silence.
segmenter.pushVolumeLevel(0.0, t0 + 301);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000);
segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000 + 2000);
expect(onSpeechEnd).toHaveBeenCalledTimes(1);
expect(detectingChanges).toEqual([true, false, true, false]);
expect(speakingChanges).toEqual([true, false]);
expect(onAudioSegment).toHaveBeenCalled();
const lastCall = onAudioSegment.mock.calls.at(-1)![0];
@@ -121,4 +125,148 @@ describe("SpeechSegmenter", () => {
expect(detectingChanges).toContain(true);
expect(onSpeechStart).toHaveBeenCalledTimes(1);
});
it("keeps utterance open after fade-out and resumes speaking during silence grace", () => {
const onAudioSegment = vi.fn();
const onSpeechStart = vi.fn();
const onSpeechEnd = vi.fn();
const speakingChanges: boolean[] = [];
const segmenter = new SpeechSegmenter(
{
enableContinuousStreaming: false,
volumeThreshold: 0.2,
silenceDurationMs: 1500,
speechConfirmationMs: 120,
detectionGracePeriodMs: 400,
minChunkDurationMs: 1000, // keep everything buffered until final flush
pcmSampleRate: 1000,
},
{
onAudioSegment,
onSpeechStart,
onSpeechEnd,
onSpeakingChange: (v) => speakingChanges.push(v),
}
);
const t0 = 30_000;
// Start + confirm speaking.
segmenter.pushVolumeLevel(0.6, t0);
segmenter.pushPcmChunk(mkPcmBytes(200));
segmenter.pushVolumeLevel(0.6, t0 + 130);
segmenter.pushPcmChunk(mkPcmBytes(200));
expect(onSpeechStart).toHaveBeenCalledTimes(1);
// Drop below threshold: meter should fade, but turn remains open.
segmenter.pushVolumeLevel(0.0, t0 + 200);
segmenter.pushPcmChunk(mkPcmBytes(200));
segmenter.pushVolumeLevel(0.0, t0 + 500);
segmenter.pushPcmChunk(mkPcmBytes(200));
// Speak again before silence timeout expires: continue same utterance.
segmenter.pushVolumeLevel(0.6, t0 + 900);
segmenter.pushPcmChunk(mkPcmBytes(200));
expect(onSpeechStart).toHaveBeenCalledTimes(1);
expect(onSpeechEnd).not.toHaveBeenCalled();
// End for real.
segmenter.pushVolumeLevel(0.0, t0 + 1000);
segmenter.pushPcmChunk(mkPcmBytes(200));
segmenter.pushVolumeLevel(0.0, t0 + 2600);
segmenter.pushVolumeLevel(0.0, t0 + 4200);
expect(speakingChanges).toEqual([true, false, true, false]);
expect(onSpeechEnd).toHaveBeenCalledTimes(1);
const finalCalls = onAudioSegment.mock.calls.filter((call) => call[0].isLast);
expect(finalCalls).toHaveLength(1);
});
it("keeps confirmed speech active above release threshold even when below start threshold", () => {
const onSpeechStart = vi.fn();
const onSpeechEnd = vi.fn();
const speakingChanges: boolean[] = [];
const segmenter = new SpeechSegmenter(
{
enableContinuousStreaming: false,
volumeThreshold: 0.3,
volumeReleaseThreshold: 0.1,
silenceDurationMs: 1200,
speechConfirmationMs: 120,
detectionGracePeriodMs: 200,
minChunkDurationMs: 1000,
pcmSampleRate: 1000,
},
{
onSpeechStart,
onSpeechEnd,
onSpeakingChange: (v) => speakingChanges.push(v),
}
);
const t0 = 40_000;
// Confirm speech.
segmenter.pushVolumeLevel(0.4, t0);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.4, t0 + 130);
segmenter.pushPcmChunk(mkPcmBytes(50));
expect(onSpeechStart).toHaveBeenCalledTimes(1);
// Stay between start and release thresholds for longer than silence timeout:
// should still be considered speaking.
segmenter.pushVolumeLevel(0.15, t0 + 400);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.15, t0 + 1000);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.15, t0 + 2000);
segmenter.pushPcmChunk(mkPcmBytes(50));
expect(onSpeechEnd).not.toHaveBeenCalled();
expect(speakingChanges.at(-1)).toBe(true);
// Now actually drop below release threshold and allow silence timeout.
segmenter.pushVolumeLevel(0.0, t0 + 2001);
segmenter.pushPcmChunk(mkPcmBytes(50));
segmenter.pushVolumeLevel(0.0, t0 + 3405);
segmenter.pushVolumeLevel(0.0, t0 + 4706);
expect(onSpeechEnd).toHaveBeenCalledTimes(1);
expect(speakingChanges.at(-1)).toBe(false);
});
it("ignores brief confirmed dips below release threshold without flip-flopping speaking", () => {
const speakingChanges: boolean[] = [];
const segmenter = new SpeechSegmenter(
{
enableContinuousStreaming: false,
volumeThreshold: 0.2,
volumeReleaseThreshold: 0.08,
confirmedDropGracePeriodMs: 250,
silenceDurationMs: 1500,
speechConfirmationMs: 120,
detectionGracePeriodMs: 200,
minChunkDurationMs: 1000,
pcmSampleRate: 1000,
},
{
onSpeakingChange: (v) => speakingChanges.push(v),
}
);
const t0 = 50_000;
segmenter.pushVolumeLevel(0.4, t0);
segmenter.pushVolumeLevel(0.4, t0 + 130); // confirm speaking
// Two short dips below release threshold; both shorter than debounce.
segmenter.pushVolumeLevel(0.05, t0 + 300);
segmenter.pushVolumeLevel(0.09, t0 + 500);
segmenter.pushVolumeLevel(0.05, t0 + 620);
segmenter.pushVolumeLevel(0.1, t0 + 760);
expect(speakingChanges).toEqual([true]);
});
});
packages/app/src/voice/speech-segmenter.ts
+124 -71
View File
@@ -5,6 +5,8 @@ export type SpeechSegment = { audioData: string; isLast: boolean };
export type SpeechSegmenterConfig = {
enableContinuousStreaming: boolean;
volumeThreshold: number; // 0..1
volumeReleaseThreshold?: number; // 0..1; lower threshold used to keep active speech from dropping out
confirmedDropGracePeriodMs?: number; // ms dip debounce before leaving confirmed speaking
silenceDurationMs: number;
speechConfirmationMs: number;
detectionGracePeriodMs: number;
@@ -58,6 +60,7 @@ export class SpeechSegmenter {
private speechDetectionStartMs: number | null = null;
private detectionSilenceStartMs: number | null = null;
private confirmedDropStartMs: number | null = null;
private silenceStartMs: number | null = null;
private isSpeaking = false;
private isDetecting = false;
@@ -67,8 +70,14 @@ export class SpeechSegmenter {
private minChunkBytes: number;
constructor(config: SpeechSegmenterConfig, callbacks: SpeechSegmenterCallbacks = {}) {
const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold(
config.volumeThreshold,
config.volumeReleaseThreshold
);
this.config = {
...config,
volumeReleaseThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriodMs ?? 250,
minChunkDurationMs: config.minChunkDurationMs ?? 1000,
pcmSampleRate: config.pcmSampleRate ?? 16000,
pcmChannels: config.pcmChannels ?? 1,
@@ -81,8 +90,14 @@ export class SpeechSegmenter {
}
updateConfig(next: SpeechSegmenterConfig): void {
const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold(
next.volumeThreshold,
next.volumeReleaseThreshold
);
this.config = {
...next,
volumeReleaseThreshold,
confirmedDropGracePeriodMs: next.confirmedDropGracePeriodMs ?? this.config.confirmedDropGracePeriodMs,
minChunkDurationMs: next.minChunkDurationMs ?? this.config.minChunkDurationMs,
pcmSampleRate: next.pcmSampleRate ?? this.config.pcmSampleRate,
pcmChannels: next.pcmChannels ?? this.config.pcmChannels,
@@ -107,11 +122,16 @@ export class SpeechSegmenter {
return this.isDetecting;
}
getVolumeReleaseThreshold(): number {
return this.config.volumeReleaseThreshold;
}
reset(): void {
this.audioBuffer = [];
this.bufferedBytes = 0;
this.speechDetectionStartMs = null;
this.detectionSilenceStartMs = null;
this.confirmedDropStartMs = null;
this.silenceStartMs = null;
this.isSpeaking = false;
this.isDetecting = false;
@@ -170,79 +190,103 @@ export class SpeechSegmenter {
return;
}
const speechDetected = volume > this.config.volumeThreshold;
const speechDetectedForStart = volume > this.config.volumeThreshold;
const speechDetectedForContinue = volume > this.config.volumeReleaseThreshold;
if (speechDetected && !this.isSpeaking && !this.speechConfirmed) {
// Initial detection phase.
if (this.speechDetectionStartMs === null) {
this.speechDetectionStartMs = nowMs;
this.detectionSilenceStartMs = null;
this.audioBuffer = [];
this.bufferedBytes = 0;
this.setDetecting(true);
return;
}
// Volume is back above threshold - reset grace period.
this.detectionSilenceStartMs = null;
const speechDuration = nowMs - this.speechDetectionStartMs;
if (speechDuration >= this.config.speechConfirmationMs) {
this.speechConfirmed = true;
this.silenceStartMs = null;
this.setDetecting(false);
this.setSpeaking(true);
this.callbacks.onSpeechStart?.();
}
return;
}
if (speechDetected && this.isSpeaking && this.speechConfirmed) {
// Continuing confirmed speech.
this.silenceStartMs = null;
return;
}
if (!speechDetected && !this.speechConfirmed && this.speechDetectionStartMs !== null) {
// Volume dropped during detection phase - apply grace period.
if (this.detectionSilenceStartMs === null) {
this.detectionSilenceStartMs = nowMs;
return;
}
const graceDuration = nowMs - this.detectionSilenceStartMs;
if (graceDuration >= this.config.detectionGracePeriodMs) {
// Cancel detection.
this.speechDetectionStartMs = null;
this.detectionSilenceStartMs = null;
this.audioBuffer = [];
this.bufferedBytes = 0;
this.setDetecting(false);
}
return;
}
if (!speechDetected && this.isSpeaking && this.speechConfirmed) {
// Potential speech END.
if (this.silenceStartMs === null) {
this.silenceStartMs = nowMs;
return;
}
const silenceDuration = nowMs - this.silenceStartMs;
if (silenceDuration >= this.config.silenceDurationMs) {
// Speech END confirmed.
this.speechConfirmed = false;
this.speechDetectionStartMs = null;
this.silenceStartMs = null;
this.setSpeaking(false);
this.callbacks.onSpeechEnd?.();
if (this.audioBuffer.length === 0) {
// Important: downstream expects an explicit segment boundary marker.
this.callbacks.onAudioSegment?.({ audioData: "", isLast: true });
} else {
this.flush(true);
if (!this.speechConfirmed) {
if (speechDetectedForStart) {
// Initial detection phase.
this.confirmedDropStartMs = null;
if (this.speechDetectionStartMs === null) {
this.speechDetectionStartMs = nowMs;
this.detectionSilenceStartMs = null;
this.audioBuffer = [];
this.bufferedBytes = 0;
this.setDetecting(true);
this.setSpeaking(true);
return;
}
// Volume is back above threshold - reset grace period.
this.detectionSilenceStartMs = null;
this.setSpeaking(true);
const speechDuration = nowMs - this.speechDetectionStartMs;
if (speechDuration >= this.config.speechConfirmationMs) {
this.speechConfirmed = true;
this.silenceStartMs = null;
this.setDetecting(false);
this.callbacks.onSpeechStart?.();
}
return;
}
if (this.speechDetectionStartMs !== null) {
// Volume dropped during detection phase - apply grace period.
if (this.detectionSilenceStartMs === null) {
this.detectionSilenceStartMs = nowMs;
this.setSpeaking(false);
return;
}
const graceDuration = nowMs - this.detectionSilenceStartMs;
if (graceDuration >= this.config.detectionGracePeriodMs) {
// Cancel detection.
this.speechDetectionStartMs = null;
this.detectionSilenceStartMs = null;
this.confirmedDropStartMs = null;
this.audioBuffer = [];
this.bufferedBytes = 0;
this.setDetecting(false);
this.setSpeaking(false);
}
}
return;
}
if (speechDetectedForContinue) {
// Continuing confirmed speech (or resuming during silence grace).
this.confirmedDropStartMs = null;
this.silenceStartMs = null;
this.setDetecting(false);
this.setSpeaking(true);
return;
}
if (this.confirmedDropStartMs === null) {
this.confirmedDropStartMs = nowMs;
return;
}
const confirmedDropDuration = nowMs - this.confirmedDropStartMs;
if (confirmedDropDuration < this.config.confirmedDropGracePeriodMs) {
return;
}
// Potential speech END while utterance remains open.
if (this.silenceStartMs === null) {
this.silenceStartMs = nowMs;
this.setDetecting(true);
this.setSpeaking(false);
return;
}
const silenceDuration = nowMs - this.silenceStartMs;
if (silenceDuration >= this.config.silenceDurationMs) {
// Speech END confirmed.
this.speechConfirmed = false;
this.speechDetectionStartMs = null;
this.detectionSilenceStartMs = null;
this.confirmedDropStartMs = null;
this.silenceStartMs = null;
this.setDetecting(false);
this.setSpeaking(false);
this.callbacks.onSpeechEnd?.();
if (this.audioBuffer.length === 0) {
// Important: downstream expects an explicit segment boundary marker.
this.callbacks.onAudioSegment?.({ audioData: "", isLast: true });
} else {
this.flush(true);
}
}
}
@@ -282,4 +326,13 @@ export class SpeechSegmenter {
1000;
this.minChunkBytes = Math.round(this.pcmBytesPerMs * this.config.minChunkDurationMs);
}
private resolveVolumeReleaseThreshold(
volumeThreshold: number,
configured?: number
): number {
const defaultReleaseThreshold = volumeThreshold * 0.4;
const raw = configured ?? defaultReleaseThreshold;
return Math.max(0, Math.min(raw, volumeThreshold));
}
}