From 0d60588431ad0b939511e4852f13120fdb48576d Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 11 Feb 2026 11:03:51 +0700 Subject: [PATCH] Update files --- packages/app/src/components/volume-meter.tsx | 13 +- packages/app/src/contexts/voice-context.tsx | 53 ++- .../hooks/use-speechmatics-audio.native.ts | 5 + .../src/hooks/use-speechmatics-audio.web.ts | 45 +- .../src/screens/agent/agent-ready-screen.tsx | 12 +- .../src/screens/agent/draft-agent-screen.tsx | 7 +- .../app/src/voice/realtime-voice-config.ts | 11 +- .../app/src/voice/speech-segmenter.test.ts | 148 ++++++ packages/app/src/voice/speech-segmenter.ts | 195 +++++--- packages/server/package.json | 1 + .../scripts/generate-sherpa-tts-matrix.ts | 433 ++++++++++++++++++ .../src/server/agent/agent-manager.test.ts | 90 +++- .../server/src/server/agent/agent-manager.ts | 50 +- .../agent/agent-metadata-generator.test.ts | 4 +- .../agent/providers/claude-agent.test.ts | 40 ++ .../providers/codex-app-server-agent.test.ts | 83 ++-- .../daemon-e2e/git-operations.e2e.test.ts | 7 +- .../server/daemon-e2e/tool-calls.e2e.test.ts | 126 ++++- packages/server/src/server/session.ts | 238 ++++------ .../server/speech/providers/local/config.ts | 8 +- .../providers/local/sherpa/model-catalog.ts | 2 +- .../speech/speech-config-resolver.test.ts | 9 +- packages/website/src/routes/docs/voice.tsx | 4 +- 23 files changed, 1264 insertions(+), 320 deletions(-) create mode 100644 packages/server/scripts/generate-sherpa-tts-matrix.ts diff --git a/packages/app/src/components/volume-meter.tsx b/packages/app/src/components/volume-meter.tsx index dd55dd98b..6c87be3af 100644 --- a/packages/app/src/components/volume-meter.tsx +++ b/packages/app/src/components/volume-meter.tsx @@ -24,7 +24,6 @@ interface VolumeMeterProps { export function VolumeMeter({ volume, isMuted = false, - isDetecting = false, isSpeaking = false, orientation = "vertical", variant = "default", @@ -146,9 +145,9 @@ export function VolumeMeter({ // Create animated styles unconditionally at top level const line1Style = useAnimatedStyle(() => { - const isActive = isDetecting || isSpeaking; + const isActive = isSpeaking; const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5; - const volumeBoost = isMuted ? 0 : volume * 0.3; + const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3; return { height: line1Height.value * (isMuted || volume > 0.001 ? 1 : line1Pulse.value), opacity: baseOpacity + volumeBoost, @@ -156,9 +155,9 @@ export function VolumeMeter({ }); const line2Style = useAnimatedStyle(() => { - const isActive = isDetecting || isSpeaking; + const isActive = isSpeaking; const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5; - const volumeBoost = isMuted ? 0 : volume * 0.3; + const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3; return { height: line2Height.value * (isMuted || volume > 0.001 ? 1 : line2Pulse.value), opacity: baseOpacity + volumeBoost, @@ -166,9 +165,9 @@ export function VolumeMeter({ }); const line3Style = useAnimatedStyle(() => { - const isActive = isDetecting || isSpeaking; + const isActive = isSpeaking; const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5; - const volumeBoost = isMuted ? 0 : volume * 0.3; + const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3; return { height: line3Height.value * (isMuted || volume > 0.001 ? 1 : line3Pulse.value), opacity: baseOpacity + volumeBoost, diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index d03da889d..c27c815e3 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -63,10 +63,15 @@ export function VoiceProvider({ children }: VoiceProviderProps) { const lastVoiceModeSyncedClientRef = useRef(null); const voiceTransportReadyRef = useRef(false); const voiceResyncInFlightRef = useRef(false); + const silenceGraceStartMsRef = useRef(null); + const vadStateRef = useRef<{ isDetecting: boolean; isSpeaking: boolean }>({ + isDetecting: false, + isSpeaking: false, + }); const realtimeAudio = useSpeechmaticsAudio({ onSpeechStart: () => { - console.log("[Voice] Speech detected"); + console.log("[Voice] Segment started (speech confirmed)"); // Stop audio playback if playing const session = realtimeSessionRef.current; const sessionAudioPlayer = session?.audioPlayer ?? null; @@ -93,7 +98,16 @@ export function VoiceProvider({ children }: VoiceProviderProps) { } }, onSpeechEnd: () => { - console.log("[Voice] Speech ended"); + const silenceMs = + silenceGraceStartMsRef.current === null + ? null + : Date.now() - silenceGraceStartMsRef.current; + if (silenceMs === null) { + console.log("[Voice] Segment finalized"); + } else { + console.log("[Voice] Segment finalized", { silenceMs }); + } + silenceGraceStartMsRef.current = null; }, onAudioSegment: ({ audioData, isLast }) => { if (!voiceTransportReadyRef.current) { @@ -134,6 +148,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) { } }, volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold, + confirmedDropGracePeriod: REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs, silenceDuration: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs, speechConfirmationDuration: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs, detectionGracePeriod: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs, @@ -143,6 +158,40 @@ export function VoiceProvider({ children }: VoiceProviderProps) { realtimeSessionRef.current = activeSession; }, [activeSession]); + useEffect(() => { + const next = { + isDetecting: realtimeAudio.isDetecting, + isSpeaking: realtimeAudio.isSpeaking, + }; + const prev = vadStateRef.current; + + if (!isVoiceMode) { + silenceGraceStartMsRef.current = null; + vadStateRef.current = next; + return; + } + + // Confirmed speech dropped below threshold: grace window starts. + if (prev.isSpeaking && !next.isSpeaking && next.isDetecting) { + silenceGraceStartMsRef.current = Date.now(); + console.log("[Voice] Grace started (speech dropped below threshold)"); + } + + // User resumed speaking before grace timeout elapsed. + if (!prev.isSpeaking && next.isSpeaking && silenceGraceStartMsRef.current !== null) { + const resumedAfterMs = Date.now() - silenceGraceStartMsRef.current; + console.log("[Voice] Speech resumed during grace", { resumedAfterMs }); + silenceGraceStartMsRef.current = null; + } + + // Fully idle (neither detecting nor speaking). + if (!next.isDetecting && !next.isSpeaking) { + silenceGraceStartMsRef.current = null; + } + + vadStateRef.current = next; + }, [isVoiceMode, realtimeAudio.isDetecting, realtimeAudio.isSpeaking]); + useEffect(() => { const connected = activeSession?.connection.isConnected ?? false; const client = activeSession?.client ?? null; diff --git a/packages/app/src/hooks/use-speechmatics-audio.native.ts b/packages/app/src/hooks/use-speechmatics-audio.native.ts index 8c8b9d59e..144f3a8e9 100644 --- a/packages/app/src/hooks/use-speechmatics-audio.native.ts +++ b/packages/app/src/hooks/use-speechmatics-audio.native.ts @@ -19,6 +19,8 @@ export interface SpeechmaticsAudioConfig { /** When true, stream microphone PCM continuously without VAD gating. */ enableContinuousStreaming?: boolean; volumeThreshold: number; // Volume threshold for speech detection (0-1) + /** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */ + confirmedDropGracePeriod?: number; silenceDuration: number; // ms of silence before ending segment speechConfirmationDuration: number; // ms of sustained speech before confirming detectionGracePeriod: number; // ms grace period for volume dips during detection @@ -83,6 +85,7 @@ export function useSpeechmaticsAudio( { enableContinuousStreaming, volumeThreshold: config.volumeThreshold, + confirmedDropGracePeriodMs: config.confirmedDropGracePeriod, silenceDurationMs: config.silenceDuration, speechConfirmationMs: config.speechConfirmationDuration, detectionGracePeriodMs: config.detectionGracePeriod, @@ -101,6 +104,7 @@ export function useSpeechmaticsAudio( segmenterRef.current?.updateConfig({ enableContinuousStreaming, volumeThreshold: config.volumeThreshold, + confirmedDropGracePeriodMs: config.confirmedDropGracePeriod, silenceDurationMs: config.silenceDuration, speechConfirmationMs: config.speechConfirmationDuration, detectionGracePeriodMs: config.detectionGracePeriod, @@ -108,6 +112,7 @@ export function useSpeechmaticsAudio( }, [ enableContinuousStreaming, config.volumeThreshold, + config.confirmedDropGracePeriod, config.silenceDuration, config.speechConfirmationDuration, config.detectionGracePeriod, diff --git a/packages/app/src/hooks/use-speechmatics-audio.web.ts b/packages/app/src/hooks/use-speechmatics-audio.web.ts index 6bf605e06..d05661f5a 100644 --- a/packages/app/src/hooks/use-speechmatics-audio.web.ts +++ b/packages/app/src/hooks/use-speechmatics-audio.web.ts @@ -10,6 +10,8 @@ export interface SpeechmaticsAudioConfig { /** When true, stream microphone PCM continuously without VAD gating. */ enableContinuousStreaming?: boolean; volumeThreshold: number; // 0-1 + /** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */ + confirmedDropGracePeriod?: number; silenceDuration: number; // ms of silence before ending segment speechConfirmationDuration: number; // ms of sustained speech before confirming detectionGracePeriod: number; // ms grace period for volume dips during detection @@ -100,6 +102,7 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat { enableContinuousStreaming, volumeThreshold: config.volumeThreshold, + confirmedDropGracePeriodMs: config.confirmedDropGracePeriod, silenceDurationMs: config.silenceDuration, speechConfirmationMs: config.speechConfirmationDuration, detectionGracePeriodMs: config.detectionGracePeriod, @@ -118,6 +121,7 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat segmenterRef.current?.updateConfig({ enableContinuousStreaming, volumeThreshold: config.volumeThreshold, + confirmedDropGracePeriodMs: config.confirmedDropGracePeriod, silenceDurationMs: config.silenceDuration, speechConfirmationMs: config.speechConfirmationDuration, detectionGracePeriodMs: config.detectionGracePeriod, @@ -125,6 +129,7 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat }, [ enableContinuousStreaming, config.volumeThreshold, + config.confirmedDropGracePeriod, config.silenceDuration, config.speechConfirmationDuration, config.detectionGracePeriod, @@ -151,6 +156,21 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat isMutedRef.current = isMuted; }, [isMuted]); + const vadConfigRef = useRef({ + volumeThreshold: config.volumeThreshold, + }); + useEffect(() => { + vadConfigRef.current = { + volumeThreshold: config.volumeThreshold, + }; + }, [config.volumeThreshold]); + + const vadLogRef = useRef({ + lastLogMs: 0, + lastSpeaking: false, + lastDetecting: false, + }); + // Update segment duration timer useEffect(() => { if (!isDetecting && !isSpeaking) { @@ -288,6 +308,30 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat const nowMs = Date.now(); segmenterRef.current?.pushVolumeLevel(normalized, nowMs); + const segmenter = segmenterRef.current; + if (segmenter) { + const speakingNow = segmenter.getIsSpeaking(); + const detectingNow = segmenter.getIsDetecting(); + const shouldLog = + nowMs - vadLogRef.current.lastLogMs >= 150 || + speakingNow !== vadLogRef.current.lastSpeaking || + detectingNow !== vadLogRef.current.lastDetecting; + if (shouldLog) { + const threshold = vadConfigRef.current.volumeThreshold; + const releaseThreshold = segmenter.getVolumeReleaseThreshold(); + console.log("[Voice][Web][VAD] level", { + volume: Number(normalized.toFixed(3)), + threshold: Number(threshold.toFixed(3)), + releaseThreshold: Number(releaseThreshold.toFixed(3)), + confirmedDropGraceMs: config.confirmedDropGracePeriod ?? 250, + isSpeaking: speakingNow, + isDetecting: detectingNow, + }); + vadLogRef.current.lastLogMs = nowMs; + vadLogRef.current.lastSpeaking = speakingNow; + vadLogRef.current.lastDetecting = detectingNow; + } + } const pcm16 = resampleToPcm16(input, context.sampleRate, 16000); const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength); @@ -347,4 +391,3 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat [start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration] ); } - diff --git a/packages/app/src/screens/agent/agent-ready-screen.tsx b/packages/app/src/screens/agent/agent-ready-screen.tsx index 62d56f2b6..a5df16116 100644 --- a/packages/app/src/screens/agent/agent-ready-screen.tsx +++ b/packages/app/src/screens/agent/agent-ready-screen.tsx @@ -553,7 +553,10 @@ function AgentScreenContent({ if (!isConnected) { return; } - if (!needsAuthoritativeSync) { + // On native clients, daemon stream forwarding is focused-agent only, so switching + // agents can leave timeline gaps unless we explicitly request a snapshot. + const shouldSyncOnEntry = needsAuthoritativeSync || Platform.OS !== "web"; + if (!shouldSyncOnEntry) { return; } @@ -563,7 +566,12 @@ function AgentScreenContent({ error, }); }); - }, [resolvedAgentId, ensureAgentIsInitialized, isConnected, needsAuthoritativeSync]); + }, [ + resolvedAgentId, + ensureAgentIsInitialized, + isConnected, + needsAuthoritativeSync, + ]); useEffect(() => { // Clear stale resolution state when route target changes. diff --git a/packages/app/src/screens/agent/draft-agent-screen.tsx b/packages/app/src/screens/agent/draft-agent-screen.tsx index 613e638bd..1d9738f5e 100644 --- a/packages/app/src/screens/agent/draft-agent-screen.tsx +++ b/packages/app/src/screens/agent/draft-agent-screen.tsx @@ -400,6 +400,7 @@ export function DraftAgentScreen({ const checkoutQueryError = checkoutStatusQuery.error instanceof Error ? checkoutStatusQuery.error.message : null; const checkoutPayloadError = checkout?.error ? checkout.error.message : null; + const isGitDirectory = checkoutStatusQuery.isSuccess && checkout?.isGit === true; const isNonGitDirectory = Boolean(trimmedWorkingDir) && @@ -430,7 +431,7 @@ export function DraftAgentScreen({ const isCreateWorktree = worktreeMode === "create"; const isAttachWorktree = worktreeMode === "attach"; - const worktreeListRoot = checkout?.repoRoot ?? trimmedWorkingDir; + const worktreeListRoot = checkout?.isGit ? checkout.repoRoot : ""; const worktreeListQuery = useQuery({ queryKey: ["paseoWorktreeList", selectedServerId, worktreeListRoot], queryFn: async () => { @@ -448,7 +449,8 @@ export function DraftAgentScreen({ return payload.worktrees ?? []; }, enabled: - Boolean(worktreeListRoot || trimmedWorkingDir) && + isGitDirectory && + Boolean(worktreeListRoot) && !repoAvailabilityError && Boolean(sessionClient) && isConnected && @@ -551,6 +553,7 @@ export function DraftAgentScreen({ }, enabled: isCreateWorktree && + isGitDirectory && !isNonGitDirectory && Boolean(baseBranch) && Boolean(trimmedWorkingDir) && diff --git a/packages/app/src/voice/realtime-voice-config.ts b/packages/app/src/voice/realtime-voice-config.ts index e18cf4c97..95d866ac9 100644 --- a/packages/app/src/voice/realtime-voice-config.ts +++ b/packages/app/src/voice/realtime-voice-config.ts @@ -1,7 +1,12 @@ export const REALTIME_VOICE_VAD_CONFIG = { - volumeThreshold: 0.18, - silenceDurationMs: 2200, + // Headset mics often report lower input volume than built-in mics. + // Keep threshold low enough to avoid requiring loud speech. + volumeThreshold: 0.12, + // Short staggered grace before we visually/VAD-deactivate speaking. + // Prevents fade-out between normal intra-word pauses. + confirmedDropGracePeriodMs: 1000, + // Keep turns open longer so brief "thinking pauses" don't end speech too early. + silenceDurationMs: 2000, speechConfirmationMs: 120, detectionGracePeriodMs: 700, } as const; - diff --git a/packages/app/src/voice/speech-segmenter.test.ts b/packages/app/src/voice/speech-segmenter.test.ts index a02428a26..ac9a6c386 100644 --- a/packages/app/src/voice/speech-segmenter.test.ts +++ b/packages/app/src/voice/speech-segmenter.test.ts @@ -68,20 +68,24 @@ describe("SpeechSegmenter", () => { segmenter.pushVolumeLevel(0.5, t0); segmenter.pushPcmChunk(mkPcmBytes(50)); expect(detectingChanges).toEqual([true]); + expect(speakingChanges).toEqual([true]); expect(onSpeechStart).not.toHaveBeenCalled(); // Confirm speech. segmenter.pushVolumeLevel(0.5, t0 + 300); segmenter.pushPcmChunk(mkPcmBytes(50)); expect(onSpeechStart).toHaveBeenCalledTimes(1); + expect(detectingChanges).toEqual([true, false]); expect(speakingChanges).toEqual([true]); // End speech after silence. segmenter.pushVolumeLevel(0.0, t0 + 301); segmenter.pushPcmChunk(mkPcmBytes(50)); segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000); + segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000 + 2000); expect(onSpeechEnd).toHaveBeenCalledTimes(1); + expect(detectingChanges).toEqual([true, false, true, false]); expect(speakingChanges).toEqual([true, false]); expect(onAudioSegment).toHaveBeenCalled(); const lastCall = onAudioSegment.mock.calls.at(-1)![0]; @@ -121,4 +125,148 @@ describe("SpeechSegmenter", () => { expect(detectingChanges).toContain(true); expect(onSpeechStart).toHaveBeenCalledTimes(1); }); + + it("keeps utterance open after fade-out and resumes speaking during silence grace", () => { + const onAudioSegment = vi.fn(); + const onSpeechStart = vi.fn(); + const onSpeechEnd = vi.fn(); + const speakingChanges: boolean[] = []; + + const segmenter = new SpeechSegmenter( + { + enableContinuousStreaming: false, + volumeThreshold: 0.2, + silenceDurationMs: 1500, + speechConfirmationMs: 120, + detectionGracePeriodMs: 400, + minChunkDurationMs: 1000, // keep everything buffered until final flush + pcmSampleRate: 1000, + }, + { + onAudioSegment, + onSpeechStart, + onSpeechEnd, + onSpeakingChange: (v) => speakingChanges.push(v), + } + ); + + const t0 = 30_000; + + // Start + confirm speaking. + segmenter.pushVolumeLevel(0.6, t0); + segmenter.pushPcmChunk(mkPcmBytes(200)); + segmenter.pushVolumeLevel(0.6, t0 + 130); + segmenter.pushPcmChunk(mkPcmBytes(200)); + expect(onSpeechStart).toHaveBeenCalledTimes(1); + + // Drop below threshold: meter should fade, but turn remains open. + segmenter.pushVolumeLevel(0.0, t0 + 200); + segmenter.pushPcmChunk(mkPcmBytes(200)); + segmenter.pushVolumeLevel(0.0, t0 + 500); + segmenter.pushPcmChunk(mkPcmBytes(200)); + + // Speak again before silence timeout expires: continue same utterance. + segmenter.pushVolumeLevel(0.6, t0 + 900); + segmenter.pushPcmChunk(mkPcmBytes(200)); + expect(onSpeechStart).toHaveBeenCalledTimes(1); + expect(onSpeechEnd).not.toHaveBeenCalled(); + + // End for real. + segmenter.pushVolumeLevel(0.0, t0 + 1000); + segmenter.pushPcmChunk(mkPcmBytes(200)); + segmenter.pushVolumeLevel(0.0, t0 + 2600); + segmenter.pushVolumeLevel(0.0, t0 + 4200); + + expect(speakingChanges).toEqual([true, false, true, false]); + expect(onSpeechEnd).toHaveBeenCalledTimes(1); + const finalCalls = onAudioSegment.mock.calls.filter((call) => call[0].isLast); + expect(finalCalls).toHaveLength(1); + }); + + it("keeps confirmed speech active above release threshold even when below start threshold", () => { + const onSpeechStart = vi.fn(); + const onSpeechEnd = vi.fn(); + const speakingChanges: boolean[] = []; + + const segmenter = new SpeechSegmenter( + { + enableContinuousStreaming: false, + volumeThreshold: 0.3, + volumeReleaseThreshold: 0.1, + silenceDurationMs: 1200, + speechConfirmationMs: 120, + detectionGracePeriodMs: 200, + minChunkDurationMs: 1000, + pcmSampleRate: 1000, + }, + { + onSpeechStart, + onSpeechEnd, + onSpeakingChange: (v) => speakingChanges.push(v), + } + ); + + const t0 = 40_000; + + // Confirm speech. + segmenter.pushVolumeLevel(0.4, t0); + segmenter.pushPcmChunk(mkPcmBytes(50)); + segmenter.pushVolumeLevel(0.4, t0 + 130); + segmenter.pushPcmChunk(mkPcmBytes(50)); + expect(onSpeechStart).toHaveBeenCalledTimes(1); + + // Stay between start and release thresholds for longer than silence timeout: + // should still be considered speaking. + segmenter.pushVolumeLevel(0.15, t0 + 400); + segmenter.pushPcmChunk(mkPcmBytes(50)); + segmenter.pushVolumeLevel(0.15, t0 + 1000); + segmenter.pushPcmChunk(mkPcmBytes(50)); + segmenter.pushVolumeLevel(0.15, t0 + 2000); + segmenter.pushPcmChunk(mkPcmBytes(50)); + + expect(onSpeechEnd).not.toHaveBeenCalled(); + expect(speakingChanges.at(-1)).toBe(true); + + // Now actually drop below release threshold and allow silence timeout. + segmenter.pushVolumeLevel(0.0, t0 + 2001); + segmenter.pushPcmChunk(mkPcmBytes(50)); + segmenter.pushVolumeLevel(0.0, t0 + 3405); + segmenter.pushVolumeLevel(0.0, t0 + 4706); + + expect(onSpeechEnd).toHaveBeenCalledTimes(1); + expect(speakingChanges.at(-1)).toBe(false); + }); + + it("ignores brief confirmed dips below release threshold without flip-flopping speaking", () => { + const speakingChanges: boolean[] = []; + + const segmenter = new SpeechSegmenter( + { + enableContinuousStreaming: false, + volumeThreshold: 0.2, + volumeReleaseThreshold: 0.08, + confirmedDropGracePeriodMs: 250, + silenceDurationMs: 1500, + speechConfirmationMs: 120, + detectionGracePeriodMs: 200, + minChunkDurationMs: 1000, + pcmSampleRate: 1000, + }, + { + onSpeakingChange: (v) => speakingChanges.push(v), + } + ); + + const t0 = 50_000; + segmenter.pushVolumeLevel(0.4, t0); + segmenter.pushVolumeLevel(0.4, t0 + 130); // confirm speaking + + // Two short dips below release threshold; both shorter than debounce. + segmenter.pushVolumeLevel(0.05, t0 + 300); + segmenter.pushVolumeLevel(0.09, t0 + 500); + segmenter.pushVolumeLevel(0.05, t0 + 620); + segmenter.pushVolumeLevel(0.1, t0 + 760); + + expect(speakingChanges).toEqual([true]); + }); }); diff --git a/packages/app/src/voice/speech-segmenter.ts b/packages/app/src/voice/speech-segmenter.ts index 08b7e095f..6beae1343 100644 --- a/packages/app/src/voice/speech-segmenter.ts +++ b/packages/app/src/voice/speech-segmenter.ts @@ -5,6 +5,8 @@ export type SpeechSegment = { audioData: string; isLast: boolean }; export type SpeechSegmenterConfig = { enableContinuousStreaming: boolean; volumeThreshold: number; // 0..1 + volumeReleaseThreshold?: number; // 0..1; lower threshold used to keep active speech from dropping out + confirmedDropGracePeriodMs?: number; // ms dip debounce before leaving confirmed speaking silenceDurationMs: number; speechConfirmationMs: number; detectionGracePeriodMs: number; @@ -58,6 +60,7 @@ export class SpeechSegmenter { private speechDetectionStartMs: number | null = null; private detectionSilenceStartMs: number | null = null; + private confirmedDropStartMs: number | null = null; private silenceStartMs: number | null = null; private isSpeaking = false; private isDetecting = false; @@ -67,8 +70,14 @@ export class SpeechSegmenter { private minChunkBytes: number; constructor(config: SpeechSegmenterConfig, callbacks: SpeechSegmenterCallbacks = {}) { + const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold( + config.volumeThreshold, + config.volumeReleaseThreshold + ); this.config = { ...config, + volumeReleaseThreshold, + confirmedDropGracePeriodMs: config.confirmedDropGracePeriodMs ?? 250, minChunkDurationMs: config.minChunkDurationMs ?? 1000, pcmSampleRate: config.pcmSampleRate ?? 16000, pcmChannels: config.pcmChannels ?? 1, @@ -81,8 +90,14 @@ export class SpeechSegmenter { } updateConfig(next: SpeechSegmenterConfig): void { + const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold( + next.volumeThreshold, + next.volumeReleaseThreshold + ); this.config = { ...next, + volumeReleaseThreshold, + confirmedDropGracePeriodMs: next.confirmedDropGracePeriodMs ?? this.config.confirmedDropGracePeriodMs, minChunkDurationMs: next.minChunkDurationMs ?? this.config.minChunkDurationMs, pcmSampleRate: next.pcmSampleRate ?? this.config.pcmSampleRate, pcmChannels: next.pcmChannels ?? this.config.pcmChannels, @@ -107,11 +122,16 @@ export class SpeechSegmenter { return this.isDetecting; } + getVolumeReleaseThreshold(): number { + return this.config.volumeReleaseThreshold; + } + reset(): void { this.audioBuffer = []; this.bufferedBytes = 0; this.speechDetectionStartMs = null; this.detectionSilenceStartMs = null; + this.confirmedDropStartMs = null; this.silenceStartMs = null; this.isSpeaking = false; this.isDetecting = false; @@ -170,79 +190,103 @@ export class SpeechSegmenter { return; } - const speechDetected = volume > this.config.volumeThreshold; + const speechDetectedForStart = volume > this.config.volumeThreshold; + const speechDetectedForContinue = volume > this.config.volumeReleaseThreshold; - if (speechDetected && !this.isSpeaking && !this.speechConfirmed) { - // Initial detection phase. - if (this.speechDetectionStartMs === null) { - this.speechDetectionStartMs = nowMs; - this.detectionSilenceStartMs = null; - this.audioBuffer = []; - this.bufferedBytes = 0; - this.setDetecting(true); - return; - } - - // Volume is back above threshold - reset grace period. - this.detectionSilenceStartMs = null; - - const speechDuration = nowMs - this.speechDetectionStartMs; - if (speechDuration >= this.config.speechConfirmationMs) { - this.speechConfirmed = true; - this.silenceStartMs = null; - this.setDetecting(false); - this.setSpeaking(true); - this.callbacks.onSpeechStart?.(); - } - return; - } - - if (speechDetected && this.isSpeaking && this.speechConfirmed) { - // Continuing confirmed speech. - this.silenceStartMs = null; - return; - } - - if (!speechDetected && !this.speechConfirmed && this.speechDetectionStartMs !== null) { - // Volume dropped during detection phase - apply grace period. - if (this.detectionSilenceStartMs === null) { - this.detectionSilenceStartMs = nowMs; - return; - } - - const graceDuration = nowMs - this.detectionSilenceStartMs; - if (graceDuration >= this.config.detectionGracePeriodMs) { - // Cancel detection. - this.speechDetectionStartMs = null; - this.detectionSilenceStartMs = null; - this.audioBuffer = []; - this.bufferedBytes = 0; - this.setDetecting(false); - } - return; - } - - if (!speechDetected && this.isSpeaking && this.speechConfirmed) { - // Potential speech END. - if (this.silenceStartMs === null) { - this.silenceStartMs = nowMs; - return; - } - - const silenceDuration = nowMs - this.silenceStartMs; - if (silenceDuration >= this.config.silenceDurationMs) { - // Speech END confirmed. - this.speechConfirmed = false; - this.speechDetectionStartMs = null; - this.silenceStartMs = null; - this.setSpeaking(false); - this.callbacks.onSpeechEnd?.(); - if (this.audioBuffer.length === 0) { - // Important: downstream expects an explicit segment boundary marker. - this.callbacks.onAudioSegment?.({ audioData: "", isLast: true }); - } else { - this.flush(true); + if (!this.speechConfirmed) { + if (speechDetectedForStart) { + // Initial detection phase. + this.confirmedDropStartMs = null; + if (this.speechDetectionStartMs === null) { + this.speechDetectionStartMs = nowMs; + this.detectionSilenceStartMs = null; + this.audioBuffer = []; + this.bufferedBytes = 0; + this.setDetecting(true); + this.setSpeaking(true); + return; } + + // Volume is back above threshold - reset grace period. + this.detectionSilenceStartMs = null; + this.setSpeaking(true); + + const speechDuration = nowMs - this.speechDetectionStartMs; + if (speechDuration >= this.config.speechConfirmationMs) { + this.speechConfirmed = true; + this.silenceStartMs = null; + this.setDetecting(false); + this.callbacks.onSpeechStart?.(); + } + return; + } + + if (this.speechDetectionStartMs !== null) { + // Volume dropped during detection phase - apply grace period. + if (this.detectionSilenceStartMs === null) { + this.detectionSilenceStartMs = nowMs; + this.setSpeaking(false); + return; + } + + const graceDuration = nowMs - this.detectionSilenceStartMs; + if (graceDuration >= this.config.detectionGracePeriodMs) { + // Cancel detection. + this.speechDetectionStartMs = null; + this.detectionSilenceStartMs = null; + this.confirmedDropStartMs = null; + this.audioBuffer = []; + this.bufferedBytes = 0; + this.setDetecting(false); + this.setSpeaking(false); + } + } + return; + } + + if (speechDetectedForContinue) { + // Continuing confirmed speech (or resuming during silence grace). + this.confirmedDropStartMs = null; + this.silenceStartMs = null; + this.setDetecting(false); + this.setSpeaking(true); + return; + } + + if (this.confirmedDropStartMs === null) { + this.confirmedDropStartMs = nowMs; + return; + } + + const confirmedDropDuration = nowMs - this.confirmedDropStartMs; + if (confirmedDropDuration < this.config.confirmedDropGracePeriodMs) { + return; + } + + // Potential speech END while utterance remains open. + if (this.silenceStartMs === null) { + this.silenceStartMs = nowMs; + this.setDetecting(true); + this.setSpeaking(false); + return; + } + + const silenceDuration = nowMs - this.silenceStartMs; + if (silenceDuration >= this.config.silenceDurationMs) { + // Speech END confirmed. + this.speechConfirmed = false; + this.speechDetectionStartMs = null; + this.detectionSilenceStartMs = null; + this.confirmedDropStartMs = null; + this.silenceStartMs = null; + this.setDetecting(false); + this.setSpeaking(false); + this.callbacks.onSpeechEnd?.(); + if (this.audioBuffer.length === 0) { + // Important: downstream expects an explicit segment boundary marker. + this.callbacks.onAudioSegment?.({ audioData: "", isLast: true }); + } else { + this.flush(true); } } } @@ -282,4 +326,13 @@ export class SpeechSegmenter { 1000; this.minChunkBytes = Math.round(this.pcmBytesPerMs * this.config.minChunkDurationMs); } + + private resolveVolumeReleaseThreshold( + volumeThreshold: number, + configured?: number + ): number { + const defaultReleaseThreshold = volumeThreshold * 0.4; + const raw = configured ?? defaultReleaseThreshold; + return Math.max(0, Math.min(raw, volumeThreshold)); + } } diff --git a/packages/server/package.json b/packages/server/package.json index 811ae6eb5..9ef40f55d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -42,6 +42,7 @@ "generate:config-schema": "tsx scripts/generate-config-schema.ts", "speech:models": "tsx scripts/list-speech-models.ts", "speech:download": "tsx scripts/download-speech-models.ts", + "speech:tts:matrix": "tsx scripts/generate-sherpa-tts-matrix.ts", "speech:transcribe:local": "tsx scripts/transcribe-local-wav.ts", "test": "vitest run", "test:watch": "vitest", diff --git a/packages/server/scripts/generate-sherpa-tts-matrix.ts b/packages/server/scripts/generate-sherpa-tts-matrix.ts new file mode 100644 index 000000000..8bc52eced --- /dev/null +++ b/packages/server/scripts/generate-sherpa-tts-matrix.ts @@ -0,0 +1,433 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { createRootLogger } from "../src/server/logger.js"; +import { resolvePaseoHome } from "../src/server/paseo-home.js"; +import { float32ToPcm16le, parsePcmRateFromFormat } from "../src/server/speech/audio.js"; +import { + ensureLocalSpeechModels, + getLocalSpeechModelDir, + type LocalTtsModelId, +} from "../src/server/speech/providers/local/models.js"; +import { PocketTtsOnnxTTS } from "../src/server/speech/providers/local/pocket/pocket-tts-onnx.js"; +import { + LOCAL_TTS_MODEL_IDS, + LocalTtsModelIdSchema, +} from "../src/server/speech/providers/local/sherpa/model-catalog.js"; +import { loadSherpaOnnxNode } from "../src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.js"; + +const DEFAULT_SAMPLE_TEXT = + "Paseo keeps your coding agents in your pocket, with fast local speech and reliable remote control."; + +type ScriptOptions = { + modelsDir: string; + outputDir: string; + text: string; + speed: number; + modelIds: LocalTtsModelId[]; +}; + +type GeneratedSample = { + modelId: LocalTtsModelId; + voiceLabel: string; + speakerId: number | null; + sampleRate: number; + durationSeconds: number; + relativePath: string; +}; + +function usage(): string { + return [ + "Generate a local Sherpa/Pocket TTS sample matrix.", + "", + "Options:", + " --models-dir Directory for downloaded local speech models", + " --output-dir Directory where WAV samples are written", + " --model TTS model to include (repeatable)", + " --text Sample sentence used for every generated file", + " --speed TTS speed multiplier for Sherpa models (default: 1.0)", + " --help Show this help", + "", + `Known TTS model IDs: ${LOCAL_TTS_MODEL_IDS.join(", ")}`, + ].join("\n"); +} + +function parsePositiveNumber(raw: string, flag: string): number { + const parsed = Number.parseFloat(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${flag} must be a positive number (received: ${raw})`); + } + return parsed; +} + +function parseArgs(argv: string[]): ScriptOptions { + const paseoHome = resolvePaseoHome(); + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + + let modelsDir = process.env.PASEO_LOCAL_MODELS_DIR ?? `${paseoHome}/models/local-speech`; + let outputDir = path.resolve(process.cwd(), ".debug", "sherpa-tts-matrix", timestamp); + let text = DEFAULT_SAMPLE_TEXT; + let speed = parsePositiveNumber(process.env.PASEO_VOICE_LOCAL_TTS_SPEED ?? "1.0", "--speed"); + const requestedModelIds: LocalTtsModelId[] = []; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + + if (arg === "--help") { + // eslint-disable-next-line no-console + console.log(usage()); + process.exit(0); + } + + if (arg === "--models-dir") { + modelsDir = path.resolve(argv[i + 1] ?? modelsDir); + i += 1; + continue; + } + + if (arg === "--output-dir") { + outputDir = path.resolve(argv[i + 1] ?? outputDir); + i += 1; + continue; + } + + if (arg === "--text") { + text = argv[i + 1] ?? text; + i += 1; + continue; + } + + if (arg === "--speed") { + const raw = argv[i + 1]; + if (!raw) { + throw new Error("--speed requires a value"); + } + speed = parsePositiveNumber(raw, "--speed"); + i += 1; + continue; + } + + if (arg === "--model") { + const raw = argv[i + 1]; + if (!raw) { + throw new Error("--model requires a value"); + } + requestedModelIds.push(LocalTtsModelIdSchema.parse(raw)); + i += 1; + continue; + } + + throw new Error(`Unknown argument: ${arg}`); + } + + const trimmed = text.trim(); + if (!trimmed) { + throw new Error("Sample text cannot be empty"); + } + + const modelIds = + requestedModelIds.length > 0 + ? Array.from(new Set(requestedModelIds)) + : [...LOCAL_TTS_MODEL_IDS]; + + return { + modelsDir, + outputDir, + text: trimmed, + speed, + modelIds, + }; +} + +function pcm16MonoToWavBuffer(pcm16: Buffer, sampleRate: number): Buffer { + const channels = 1; + const bitsPerSample = 16; + const headerSize = 44; + const wav = Buffer.alloc(headerSize + pcm16.length); + const byteRate = (sampleRate * channels * bitsPerSample) / 8; + const blockAlign = (channels * bitsPerSample) / 8; + + wav.write("RIFF", 0); + wav.writeUInt32LE(36 + pcm16.length, 4); + wav.write("WAVE", 8); + wav.write("fmt ", 12); + wav.writeUInt32LE(16, 16); + wav.writeUInt16LE(1, 20); + wav.writeUInt16LE(channels, 22); + wav.writeUInt32LE(sampleRate, 24); + wav.writeUInt32LE(byteRate, 28); + wav.writeUInt16LE(blockAlign, 32); + wav.writeUInt16LE(bitsPerSample, 34); + wav.write("data", 36); + wav.writeUInt32LE(pcm16.length, 40); + pcm16.copy(wav, 44); + + return wav; +} + +function sanitizeFilePart(input: string): string { + return input.replace(/[^a-zA-Z0-9._-]/g, "-"); +} + +function toFloatSamples(audio: unknown): Float32Array { + const raw = (audio as { samples?: unknown })?.samples; + if (raw instanceof Float32Array) { + return raw; + } + if (Array.isArray(raw)) { + return Float32Array.from(raw as number[]); + } + throw new Error("Unexpected Sherpa TTS output: missing Float32 samples"); +} + +function toSampleRate(audio: unknown, fallback: number): number { + const sampleRate = (audio as { sampleRate?: unknown })?.sampleRate; + if (typeof sampleRate === "number" && Number.isFinite(sampleRate) && sampleRate > 0) { + return sampleRate; + } + return fallback; +} + +function createSherpaOfflineTts(params: { + modelId: "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16"; + modelDir: string; + speed: number; +}): { tts: any; speed: number } { + const sherpa = loadSherpaOnnxNode(); + const model = + params.modelId === "kokoro-en-v0_19" + ? { + kokoro: { + model: `${params.modelDir}/model.onnx`, + voices: `${params.modelDir}/voices.bin`, + tokens: `${params.modelDir}/tokens.txt`, + dataDir: `${params.modelDir}/espeak-ng-data`, + lengthScale: 1.0, + }, + } + : { + kitten: { + model: `${params.modelDir}/model.fp16.onnx`, + voices: `${params.modelDir}/voices.bin`, + tokens: `${params.modelDir}/tokens.txt`, + dataDir: `${params.modelDir}/espeak-ng-data`, + lengthScale: 1.0, + }, + }; + + return { + tts: new sherpa.OfflineTts({ + model, + numThreads: 2, + provider: "cpu", + maxNumSentences: 1, + }), + speed: params.speed, + }; +} + +async function readStreamToBuffer(stream: NodeJS.ReadableStream): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream as AsyncIterable) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +function toCsv(rows: GeneratedSample[]): string { + const header = [ + "model_id", + "voice_label", + "speaker_id", + "sample_rate", + "duration_seconds", + "relative_path", + ]; + + const lines = [header.join(",")]; + for (const row of rows) { + const cells = [ + row.modelId, + row.voiceLabel, + row.speakerId === null ? "" : String(row.speakerId), + String(row.sampleRate), + row.durationSeconds.toFixed(3), + row.relativePath, + ]; + + lines.push( + cells + .map((cell) => { + const escaped = cell.replace(/"/g, "\"\""); + return escaped.includes(",") ? `"${escaped}"` : escaped; + }) + .join(",") + ); + } + return `${lines.join("\n")}\n`; +} + +const logger = createRootLogger({ level: "info", format: "pretty" }); +const options = parseArgs(process.argv.slice(2)); + +logger.info( + { + modelsDir: options.modelsDir, + outputDir: options.outputDir, + modelIds: options.modelIds, + speed: options.speed, + text: options.text, + }, + "Generating TTS matrix across all model/voice combinations" +); + +await ensureLocalSpeechModels({ + modelsDir: options.modelsDir, + modelIds: options.modelIds, + autoDownload: true, + logger, +}); + +await mkdir(options.outputDir, { recursive: true }); + +const generated: GeneratedSample[] = []; + +for (const modelId of options.modelIds) { + const modelDir = getLocalSpeechModelDir(options.modelsDir, modelId); + const modelOutputDir = path.join(options.outputDir, sanitizeFilePart(modelId)); + await mkdir(modelOutputDir, { recursive: true }); + + if (modelId === "pocket-tts-onnx-int8") { + const tts = await PocketTtsOnnxTTS.create( + { + modelDir, + precision: "int8", + targetChunkMs: 50, + }, + logger + ); + const result = await tts.synthesizeSpeech(options.text); + const pcm16 = await readStreamToBuffer(result.stream); + const sampleRate = parsePcmRateFromFormat(result.format, 24000) ?? 24000; + + const voiceLabel = "default"; + const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`; + const absPath = path.join(modelOutputDir, filename); + const relPath = path.relative(options.outputDir, absPath); + + await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate)); + + generated.push({ + modelId, + voiceLabel, + speakerId: null, + sampleRate, + durationSeconds: pcm16.length / 2 / sampleRate, + relativePath: relPath, + }); + + logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample"); + continue; + } + + if (modelId !== "kokoro-en-v0_19" && modelId !== "kitten-nano-en-v0_1-fp16") { + throw new Error(`Unsupported local Sherpa TTS model for this script: ${modelId}`); + } + + const { tts, speed } = createSherpaOfflineTts({ + modelId, + modelDir, + speed: options.speed, + }); + + try { + const rawNumSpeakers = (tts as { numSpeakers?: unknown }).numSpeakers; + const numSpeakers = + typeof rawNumSpeakers === "number" && Number.isFinite(rawNumSpeakers) && rawNumSpeakers > 0 + ? Math.floor(rawNumSpeakers) + : 1; + const pad = Math.max(2, String(Math.max(0, numSpeakers - 1)).length); + const fallbackSampleRate = + typeof (tts as { sampleRate?: unknown }).sampleRate === "number" + ? Math.max(1, Math.floor((tts as { sampleRate: number }).sampleRate)) + : 24000; + + logger.info({ modelId, numSpeakers }, "Discovered Sherpa speaker count"); + + for (let sid = 0; sid < numSpeakers; sid += 1) { + const audio = (tts as { generate: (config: { text: string; sid: number; speed: number }) => unknown }).generate({ + text: options.text, + sid, + speed, + }); + const samples = toFloatSamples(audio); + const sampleRate = toSampleRate(audio, fallbackSampleRate); + const pcm16 = float32ToPcm16le(samples); + + const voiceLabel = `sid-${String(sid).padStart(pad, "0")}`; + const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`; + const absPath = path.join(modelOutputDir, filename); + const relPath = path.relative(options.outputDir, absPath); + + await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate)); + + generated.push({ + modelId, + voiceLabel, + speakerId: sid, + sampleRate, + durationSeconds: pcm16.length / 2 / sampleRate, + relativePath: relPath, + }); + + logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample"); + } + } finally { + (tts as { free?: () => void }).free?.(); + } +} + +const manifest = { + createdAt: new Date().toISOString(), + text: options.text, + speed: options.speed, + modelsDir: options.modelsDir, + outputDir: options.outputDir, + namingConvention: "/__voice-.wav", + notes: [ + "For Sherpa kitten/kokoro models, voice-label uses sid-.", + "For Pocket TTS, voice-label is default.", + ], + generated, +}; + +await writeFile( + path.join(options.outputDir, "manifest.json"), + `${JSON.stringify(manifest, null, 2)}\n` +); +await writeFile(path.join(options.outputDir, "samples.csv"), toCsv(generated)); +await writeFile( + path.join(options.outputDir, "README.txt"), + [ + "Sherpa TTS sample matrix", + "", + `Sample text: ${options.text}`, + `Speed: ${options.speed}`, + "", + "Naming convention:", + " /__voice-.wav", + " voice-label = sid- for Sherpa models", + " voice-label = default for Pocket TTS", + "", + `Total files: ${generated.length}`, + ].join("\n") +); + +logger.info( + { + outputDir: options.outputDir, + generatedCount: generated.length, + models: Array.from(new Set(generated.map((item) => item.modelId))), + }, + "TTS matrix generation complete" +); diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 277fffe69..f11b64832 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -150,7 +150,7 @@ describe("AgentManager", () => { ).rejects.toThrow("Working directory does not exist"); }); - test("resumeAgent keeps metadata config and applies systemPrompt/mcpServers overrides", async () => { + test("resumeAgentFromPersistence keeps metadata config and applies systemPrompt/mcpServers overrides", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-resume-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); @@ -211,7 +211,7 @@ describe("AgentManager", () => { }, }; - const resumed = await manager.resumeAgent(handle, { + const resumed = await manager.resumeAgentFromPersistence(handle, { cwd: workdir, systemPrompt: "new prompt", mcpServers: { @@ -243,6 +243,92 @@ describe("AgentManager", () => { }); }); + test("reloadAgentSession preserves timeline and does not force history replay", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-reload-")); + const storagePath = join(workdir, "agents"); + const storage = new AgentStorage(storagePath, logger); + + class HistoryProbeSession extends TestAgentSession { + constructor( + config: AgentSessionConfig, + private readonly historyText: string | null + ) { + super(config); + } + + async *streamHistory(): AsyncGenerator { + if (!this.historyText) { + return; + } + yield { + type: "timeline", + provider: this.provider, + item: { type: "assistant_message", text: this.historyText }, + }; + } + } + + class HistoryProbeClient implements AgentClient { + readonly provider = "codex" as const; + readonly capabilities = TEST_CAPABILITIES; + + async isAvailable(): Promise { + return true; + } + + async createSession(config: AgentSessionConfig): Promise { + return new HistoryProbeSession(config, null); + } + + async resumeSession( + handle: AgentPersistenceHandle, + overrides?: Partial + ): Promise { + const metadata = (handle.metadata ?? {}) as Partial; + const merged: AgentSessionConfig = { + ...metadata, + ...overrides, + provider: "codex", + cwd: overrides?.cwd ?? metadata.cwd ?? process.cwd(), + }; + return new HistoryProbeSession(merged, "history replay from provider"); + } + } + + const manager = new AgentManager({ + clients: { + codex: new HistoryProbeClient(), + }, + registry: storage, + logger, + idFactory: () => "00000000-0000-4000-8000-000000000113", + }); + + const snapshot = await manager.createAgent({ + provider: "codex", + cwd: workdir, + }); + + await manager.appendTimelineItem(snapshot.id, { + type: "assistant_message", + text: "keep this timeline in memory", + }); + await manager.hydrateTimelineFromProvider(snapshot.id); + const beforeReload = manager.getTimeline(snapshot.id); + expect(beforeReload).toHaveLength(1); + + await manager.reloadAgentSession(snapshot.id, { + systemPrompt: "reloaded prompt", + }); + const afterReload = manager.getTimeline(snapshot.id); + expect(afterReload).toEqual(beforeReload); + + // If reload resets historyPrimed, this would replay provider history and append another item. + await manager.hydrateTimelineFromProvider(snapshot.id); + const afterHydrate = manager.getTimeline(snapshot.id); + expect(afterHydrate).toEqual(beforeReload); + }); + test("createAgent fails when generated agent ID is not a UUID", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 66a775794..18caed02a 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -371,7 +371,9 @@ export class AgentManager { ); } - async resumeAgent( + // Reconstruct an agent from provider persistence. Callers should explicitly + // hydrate timeline history after resume. + async resumeAgentFromPersistence( handle: AgentPersistenceHandle, overrides?: Partial, agentId?: string, @@ -384,7 +386,7 @@ export class AgentManager { ): Promise { const resolvedAgentId = validateAgentId( agentId ?? this.idFactory(), - "resumeAgent" + "resumeAgentFromPersistence" ); const metadata = (handle.metadata ?? {}) as Partial; const mergedConfig = { @@ -407,11 +409,18 @@ export class AgentManager { ); } - async refreshAgentFromPersistence( + // Hot-reload an active agent session with config overrides while preserving + // in-memory timeline state. + async reloadAgentSession( agentId: string, overrides?: Partial ): Promise { const existing = this.requireAgent(agentId); + const preservedTimeline = [...existing.timeline]; + const preservedHistoryPrimed = existing.historyPrimed; + const preservedLastUsage = existing.lastUsage; + const preservedLastError = existing.lastError; + const preservedAttention = existing.attention; const handle = existing.persistence; const provider = handle?.provider ?? existing.provider; const client = this.requireClient(provider); @@ -437,12 +446,17 @@ export class AgentManager { ); } - // Preserve existing labels during refresh + // Preserve existing labels and timeline during reload. return this.registerSession(session, normalizedConfig, agentId, { labels: existing.labels, createdAt: existing.createdAt, updatedAt: existing.updatedAt, lastUserMessageAt: existing.lastUserMessageAt, + timeline: preservedTimeline, + historyPrimed: preservedHistoryPrimed, + lastUsage: preservedLastUsage, + lastError: preservedLastError, + attention: preservedAttention, }); } @@ -844,9 +858,9 @@ export class AgentManager { return iterator.done ? null : iterator.value; } - async primeAgentHistory(agentId: string): Promise { + async hydrateTimelineFromProvider(agentId: string): Promise { const agent = this.requireAgent(agentId); - await this.primeHistory(agent); + await this.hydrateTimeline(agent); } private getLastAssistantMessage(agentId: string): string | null { @@ -1033,6 +1047,11 @@ export class AgentManager { updatedAt?: Date; lastUserMessageAt?: Date | null; labels?: Record; + timeline?: AgentTimelineItem[]; + historyPrimed?: boolean; + lastUsage?: AgentUsage; + lastError?: string; + attention?: AttentionState; } ): Promise { const resolvedAgentId = validateAgentId(agentId, "registerSession"); @@ -1056,11 +1075,22 @@ export class AgentManager { currentModeId: null, pendingPermissions: new Map(), pendingRun: null, - timeline: [], + timeline: options?.timeline ? [...options.timeline] : [], persistence: attachPersistenceCwd(session.describePersistence(), config.cwd), - historyPrimed: false, + historyPrimed: options?.historyPrimed ?? false, lastUserMessageAt: options?.lastUserMessageAt ?? null, - attention: { requiresAttention: false }, + lastUsage: options?.lastUsage, + lastError: options?.lastError, + attention: + options?.attention != null + ? options.attention.requiresAttention + ? { + requiresAttention: true, + attentionReason: options.attention.attentionReason, + attentionTimestamp: new Date(options.attention.attentionTimestamp), + } + : { requiresAttention: false } + : { requiresAttention: false }, internal: config.internal ?? false, labels: options?.labels ?? {}, } as ActiveManagedAgent; @@ -1144,7 +1174,7 @@ export class AgentManager { } } - private async primeHistory(agent: ActiveManagedAgent): Promise { + private async hydrateTimeline(agent: ActiveManagedAgent): Promise { if (agent.historyPrimed) { return; } diff --git a/packages/server/src/server/agent/agent-metadata-generator.test.ts b/packages/server/src/server/agent/agent-metadata-generator.test.ts index cf18f768e..94e5c16ef 100644 --- a/packages/server/src/server/agent/agent-metadata-generator.test.ts +++ b/packages/server/src/server/agent/agent-metadata-generator.test.ts @@ -82,7 +82,7 @@ function initGitRepo(repoDir: string): void { modeId: "auto", cwd: repoDir, title: "Main Agent", - }, "metadata-title-agent"); + }, "4e0a4508-e522-4fe9-8384-cf3bf889f16d"); await generateAndApplyAgentMetadata({ agentManager: manager, @@ -122,7 +122,7 @@ function initGitRepo(repoDir: string): void { modeId: "auto", cwd: worktree.worktreePath, title: "Worktree Agent", - }, "metadata-branch-agent"); + }, "32bb765d-f637-44a2-9820-f2efd5261418"); await generateAndApplyAgentMetadata({ agentManager: manager, diff --git a/packages/server/src/server/agent/providers/claude-agent.test.ts b/packages/server/src/server/agent/providers/claude-agent.test.ts index a9de83dc3..81aff983e 100644 --- a/packages/server/src/server/agent/providers/claude-agent.test.ts +++ b/packages/server/src/server/agent/providers/claude-agent.test.ts @@ -825,6 +825,7 @@ async function startAgentMcpServer(): Promise { // Wait for history file to be written const historyPaths = getClaudeHistoryPaths(cwd, handle!.sessionId); expect(await waitForHistoryFile(historyPaths)).toBe(true); + expect(await waitForHistoryContains(historyPaths, secretWord)).toBe(true); // Resume and verify context is preserved const resumed = await client.resumeSession(handle!, { cwd }); @@ -1079,6 +1080,7 @@ async function startAgentMcpServer(): Promise { const historyPaths = getClaudeHistoryPaths(cwd, sessionId!); expect(await waitForHistoryFile(historyPaths)).toBe(true); + expect(await waitForHistoryContains(historyPaths, "HYDRATION_PROOF_LINE_TWO")).toBe(true); const resumed = await client.resumeSession(handle!, { cwd }); const hydrationUpdates: StreamHydrationUpdate[] = []; @@ -1179,6 +1181,7 @@ async function startAgentMcpServer(): Promise { const historyPaths = getClaudeHistoryPaths(cwd, sessionId!); expect(await waitForHistoryFile(historyPaths)).toBe(true); + expect(await waitForHistoryContains(historyPaths, promptMarker)).toBe(true); const resumed = await client.resumeSession(handle!, { cwd }); const hydrationUpdates: StreamHydrationUpdate[] = []; @@ -1524,6 +1527,43 @@ async function waitForHistoryFile(historyPaths: string | string[], timeoutMs = 1 return candidates.some((entry) => existsSync(entry)); } +async function waitForHistoryContains( + historyPaths: string | string[], + marker: string, + timeoutMs = 15_000 +): Promise { + const candidates = Array.isArray(historyPaths) + ? Array.from(new Set(historyPaths)) + : [historyPaths]; + const deadline = Date.now() + timeoutMs; + + const hasMarker = (): boolean => { + for (const historyPath of candidates) { + if (!existsSync(historyPath)) { + continue; + } + try { + const content = readFileSync(historyPath, "utf8"); + if (content.includes(marker)) { + return true; + } + } catch { + // History file may still be mid-write; retry. + } + } + return false; + }; + + while (Date.now() < deadline) { + if (hasMarker()) { + return true; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + + return hasMarker(); +} + function rawContainsText(raw: unknown, text: string, depth = 0): boolean { if (!raw || typeof text !== "string" || !text) { return false; diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index c0f645c73..bed0acead 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -1214,11 +1214,15 @@ describe("Codex app-server provider (integration)", () => { (item) => item.type === "assistant_message" || item.type === "user_message" ) ).toBe(true); - - const response = await resumed.run( - `Respond with the exact token ${token} and stop.` + const historyIncludesToken = history.some( + (item) => + (item.type === "assistant_message" || item.type === "user_message") && + item.text.includes(token) ); - expect(response.finalText).toContain(token); + expect(historyIncludesToken).toBe(true); + + const response = await resumed.run("Reply with CONTEXT_OK and stop."); + expect(response.finalText.toLowerCase()).toContain("context_ok"); const resumedHandle = resumed.describePersistence(); expect(resumedHandle?.sessionId).toBe(handle?.sessionId); @@ -1327,34 +1331,51 @@ describe("Codex app-server provider (integration)", () => { ); let failure: string | null = null; - for await (const event of events) { - if (event.type === "permission_requested" && event.request.name === "CodexFileChange") { - sawPermission = true; - captured = event.request; - await session.respondToPermission(event.request.id, { behavior: "allow" }); - } - if ( - event.type === "permission_resolved" && - captured && - event.requestId === captured.id && - event.resolution.behavior === "allow" - ) { - sawPermissionResolved = true; - } - if (event.type === "timeline" && event.item.type === "tool_call") { - timelineItems.push(event.item); - } - if (event.type === "turn_failed") { - failure = event.error; - break; - } - if (event.type === "turn_completed") { - break; - } + try { + await Promise.race([ + (async () => { + for await (const event of events) { + if (event.type === "permission_requested" && event.request.name === "CodexFileChange") { + sawPermission = true; + captured = event.request; + await session.respondToPermission(event.request.id, { behavior: "allow" }); + } + if ( + event.type === "permission_resolved" && + captured && + event.requestId === captured.id && + event.resolution.behavior === "allow" + ) { + sawPermissionResolved = true; + } + if (event.type === "timeline" && event.item.type === "tool_call") { + timelineItems.push(event.item); + } + if (event.type === "turn_failed") { + failure = event.error; + break; + } + if (event.type === "turn_completed") { + break; + } + } + })(), + new Promise((_, reject) => + setTimeout( + () => + reject( + new Error( + "Timed out waiting for Codex file approval flow to complete" + ) + ), + 100_000 + ) + ), + ]); + } finally { + await session.close(); } - await session.close(); - if (failure) { throw new Error(failure); } @@ -1391,7 +1412,7 @@ describe("Codex app-server provider (integration)", () => { cleanup(); rmSync(cwd, { recursive: true, force: true }); } - }, 60000); + }, 120000); test.runIf(isCodexInstalled())("tool approval flow requests user input for app tools", async () => { const cleanup = useTempCodexSessionDir(); diff --git a/packages/server/src/server/daemon-e2e/git-operations.e2e.test.ts b/packages/server/src/server/daemon-e2e/git-operations.e2e.test.ts index 54ba359d4..d9f4c7485 100644 --- a/packages/server/src/server/daemon-e2e/git-operations.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/git-operations.e2e.test.ts @@ -424,7 +424,10 @@ describe("daemon E2E", () => { ); expect(completed.callId).toBeTruthy(); - expect(completed.output).toBeTruthy(); + expect(completed.detail.type).toBe("unknown"); + if (completed.detail.type === "unknown") { + expect(completed.detail.output).toBeTruthy(); + } expect(existsSync(path.join(agent.cwd, "setup-done.txt"))).toBe(true); await ctx.client.deleteAgent(agent.id); @@ -507,7 +510,7 @@ describe("daemon E2E", () => { expect(existsSync(path.join(agent.cwd, "setup-start.txt"))).toBe(true); - const output = failed.output as any; + const output = failed.detail.type === "unknown" ? failed.detail.output as any : undefined; const commands = output?.commands as any[] | undefined; expect(Array.isArray(commands)).toBe(true); expect(commands?.[0]?.exitCode).toBe(7); diff --git a/packages/server/src/server/daemon-e2e/tool-calls.e2e.test.ts b/packages/server/src/server/daemon-e2e/tool-calls.e2e.test.ts index a64e5d081..868698481 100644 --- a/packages/server/src/server/daemon-e2e/tool-calls.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/tool-calls.e2e.test.ts @@ -18,6 +18,78 @@ function tmpCwd(): string { const CODEX_TEST_MODEL = "gpt-5.1-codex-mini"; const CODEX_TEST_THINKING_OPTION_ID = "low"; +type ToolCallItem = Extract; + +function asRecord(value: unknown): Record | null { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return null; + } + return value as Record; +} + +function commandFromRawInput(value: unknown): string | null { + const record = asRecord(value); + if (!record) { + return null; + } + + const commandValue = record.command ?? record.cmd; + if (typeof commandValue === "string" && commandValue.length > 0) { + return commandValue; + } + if (Array.isArray(commandValue)) { + const parts = commandValue.filter((entry): entry is string => typeof entry === "string"); + if (parts.length > 0) { + return parts.join(" "); + } + } + return null; +} + +function commandFromToolCallDetail(detail: ToolCallItem["detail"]): string | null { + if (detail.type === "shell") { + return detail.command; + } + if (detail.type === "unknown") { + return commandFromRawInput(detail.input); + } + return null; +} + +function extractPathFromPayload(value: unknown): string | null { + const record = asRecord(value); + if (!record) { + return null; + } + + const directPath = record.path ?? record.file_path ?? record.filePath; + if (typeof directPath === "string" && directPath.length > 0) { + return directPath; + } + + if (Array.isArray(record.files)) { + for (const file of record.files) { + const fileRecord = asRecord(file); + const filePath = fileRecord?.path; + if (typeof filePath === "string" && filePath.length > 0) { + return filePath; + } + } + } + + return null; +} + +function pathFromToolCallDetail(detail: ToolCallItem["detail"]): string | null { + if (detail.type === "read" || detail.type === "edit" || detail.type === "write") { + return detail.filePath; + } + if (detail.type === "unknown") { + return extractPathFromPayload(detail.input) ?? extractPathFromPayload(detail.output); + } + return null; +} + describe("daemon E2E", () => { let ctx: DaemonTestContext; @@ -34,9 +106,9 @@ describe("daemon E2E", () => { function extractToolCalls( queue: SessionOutboundMessage[], agentId: string - ): AgentTimelineItem[] { - const byCallId = new Map(); - const noCallId: AgentTimelineItem[] = []; + ): ToolCallItem[] { + const byCallId = new Map(); + const noCallId: ToolCallItem[] = []; for (const m of queue) { if ( @@ -58,9 +130,9 @@ describe("daemon E2E", () => { } // Helper to log tool call structure in a consistent format - function logToolCall(prefix: string, tc: AgentTimelineItem): void { - if (tc.type !== "tool_call") return; - + function logToolCall(prefix: string, tc: ToolCallItem): void { + void prefix; + void tc; } test( @@ -92,10 +164,15 @@ describe("daemon E2E", () => { logToolCall("CLAUDE_READ", tc); } - const readCall = toolCalls.find((tc) => tc.type === "tool_call" && tc.name === "Read"); + const readCall = toolCalls.find((tc) => tc.name === "Read"); expect(readCall).toBeDefined(); expect(readCall?.name).toBe("Read"); - expect(readCall?.input).toBeDefined(); + expect(readCall?.detail).toBeDefined(); + const readPath = readCall ? pathFromToolCallDetail(readCall.detail) : null; + expect(readPath).toBeTruthy(); + if (readPath) { + expect(readPath).toContain("/etc/hosts"); + } await ctx.client.deleteAgent(agent.id); collector.unsubscribe(); @@ -133,13 +210,15 @@ describe("daemon E2E", () => { logToolCall("CLAUDE_BASH", tc); } - const bashCall = toolCalls.find((tc) => tc.type === "tool_call" && tc.name === "Bash"); + const bashCall = toolCalls.find((tc) => tc.name === "Bash"); expect(bashCall).toBeDefined(); expect(bashCall?.name).toBe("Bash"); - expect(bashCall?.input).toBeDefined(); - // Command text should be in input.command - const bashInput = bashCall?.input as { command?: string } | undefined; - expect(bashInput?.command).toContain("echo"); + expect(bashCall?.detail).toBeDefined(); + const command = bashCall ? commandFromToolCallDetail(bashCall.detail) : null; + expect(command).toBeTruthy(); + if (command) { + expect(command).toContain("echo"); + } await ctx.client.deleteAgent(agent.id); collector.unsubscribe(); @@ -179,9 +258,17 @@ describe("daemon E2E", () => { logToolCall("CLAUDE_EDIT", tc); } - const editCall = toolCalls.find((tc) => tc.type === "tool_call" && tc.name === "Edit"); + const editCall = toolCalls.find((tc) => tc.name === "Edit"); expect(editCall).toBeDefined(); - expect(editCall?.input).toBeDefined(); + expect(editCall?.detail).toBeDefined(); + const editPath = editCall ? pathFromToolCallDetail(editCall.detail) : null; + if (editPath) { + expect(editPath).toBe(testFile); + } else if (editCall?.detail.type === "unknown") { + expect(editCall.detail.input ?? editCall.detail.output).toBeTruthy(); + } else { + expect(editCall?.detail.type).toBe("edit"); + } await ctx.client.deleteAgent(agent.id); collector.unsubscribe(); @@ -219,15 +306,12 @@ describe("daemon E2E", () => { logToolCall("CODEX_SHELL", tc); } - const shellCalls = toolCalls.filter( - (tc) => tc.type === "tool_call" && tc.name === "shell" - ); + const shellCalls = toolCalls.filter((tc) => tc.name === "shell"); expect(shellCalls.length).toBeGreaterThan(0); const echoCall = shellCalls.find((tc) => { - const shellInput = tc.input as { command?: string } | undefined; - return typeof shellInput?.command === "string" && - shellInput.command.includes("echo"); + const command = commandFromToolCallDetail(tc.detail); + return typeof command === "string" && command.includes("echo"); }); expect(echoCall).toBeDefined(); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 96b067a99..276b73dc8 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -209,14 +209,7 @@ function deriveProjectGroupingName(projectKey: string): string { return segments[segments.length - 1] || projectKey; } -function escapeXmlText(value: string): string { - return value - .replace(/&/g, "&") - .replace(//g, ">"); -} - -type ProcessingPhase = "idle" | "transcribing" | "llm"; +type ProcessingPhase = "idle" | "transcribing"; type NormalizedGitOptions = { baseBranch?: string; @@ -392,7 +385,6 @@ export class Session { // State machine private abortController: AbortController; private processingPhase: ProcessingPhase = "idle"; - private currentStreamPromise: Promise | null = null; // Voice mode state private isVoiceMode = false; @@ -425,9 +417,6 @@ export class Session { { format: string; chunks: Buffer[] } >(); - // Conversation history - private turnIndex = 0; - // Per-session managers private readonly ttsManager: TTSManager; private readonly sttManager: STTManager; @@ -519,7 +508,7 @@ export class Session { this.defaultLocalSpeechModelIds = dictation?.localModels?.defaultModelIds && dictation.localModels.defaultModelIds.length > 0 ? [...new Set(dictation.localModels.defaultModelIds)] - : ["parakeet-tdt-0.6b-v2-int8", "pocket-tts-onnx-int8"]; + : ["parakeet-tdt-0.6b-v2-int8", "kokoro-en-v0_19"]; this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler; this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler; this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext; @@ -641,6 +630,19 @@ export class Session { } } + private hasActiveAgentRun(agentId: string | null): boolean { + if (!agentId) { + return false; + } + + const snapshot = this.agentManager.getAgent(agentId); + if (!snapshot) { + return false; + } + + return snapshot.lifecycle === "running" || Boolean(snapshot.pendingRun); + } + /** * Start streaming an agent run and forward results via the websocket broadcast */ @@ -739,6 +741,29 @@ export class Session { return; } + if ( + this.isVoiceMode && + this.voiceModeAgentId === event.agentId && + event.event.type === "permission_requested" && + isVoicePermissionAllowed(event.event.request) + ) { + const requestId = event.event.request.id; + void this.agentManager + .respondToPermission(event.agentId, requestId, { + behavior: "allow", + }) + .catch((error) => { + this.sessionLogger.warn( + { + err: error, + agentId: event.agentId, + requestId, + }, + "Failed to auto-allow speak tool permission in voice mode" + ); + }); + } + // Reduce bandwidth/CPU on mobile: only forward high-frequency agent stream events // for the focused agent, with a short grace window while backgrounded. // @@ -874,7 +899,7 @@ export class Session { const handle = toAgentPersistenceHandle(this.sessionLogger, record.persistence); let snapshot: ManagedAgent; if (handle) { - snapshot = await this.agentManager.resumeAgent( + snapshot = await this.agentManager.resumeAgentFromPersistence( handle, buildConfigOverrides(record), agentId, @@ -885,7 +910,7 @@ export class Session { snapshot = await this.agentManager.createAgent(config, agentId, { labels: record.labels }); } - await this.agentManager.primeAgentHistory(agentId); + await this.agentManager.hydrateTimelineFromProvider(agentId); return this.agentManager.getAgent(agentId) ?? snapshot; })(); @@ -1592,7 +1617,7 @@ export class Session { }; try { - const refreshed = await this.agentManager.refreshAgentFromPersistence( + const refreshed = await this.agentManager.reloadAgentSession( agentId, refreshOverrides ); @@ -1630,7 +1655,7 @@ export class Session { if (restoreAgentConfig && this.voiceModeBaseConfig) { const baseConfig = this.voiceModeBaseConfig; try { - await this.agentManager.refreshAgentFromPersistence(agentId, { + await this.agentManager.reloadAgentSession(agentId, { systemPrompt: buildVoiceModeSystemPrompt(baseConfig.systemPrompt, false), mcpServers: this.cloneMcpServers(baseConfig.mcpServers), }); @@ -2110,11 +2135,11 @@ export class Session { `Resuming agent ${handle.sessionId} (${handle.provider})` ); try { - const snapshot = await this.agentManager.resumeAgent( + const snapshot = await this.agentManager.resumeAgentFromPersistence( handle, overrides ); - await this.agentManager.primeAgentHistory(snapshot.id); + await this.agentManager.hydrateTimelineFromProvider(snapshot.id); await this.forwardAgentUpdate(snapshot); const timelineSize = this.emitAgentTimelineSnapshot(snapshot); if (requestId) { @@ -2165,7 +2190,7 @@ export class Session { if (existing) { await this.interruptAgentIfRunning(agentId); if (existing.persistence) { - snapshot = await this.agentManager.refreshAgentFromPersistence( + snapshot = await this.agentManager.reloadAgentSession( agentId ); } else { @@ -2182,14 +2207,14 @@ export class Session { `Agent ${agentId} cannot be refreshed because it lacks persistence` ); } - snapshot = await this.agentManager.resumeAgent( + snapshot = await this.agentManager.resumeAgentFromPersistence( handle, buildConfigOverrides(record), agentId, extractTimestamps(record) ); } - await this.agentManager.primeAgentHistory(agentId); + await this.agentManager.hydrateTimelineFromProvider(agentId); await this.forwardAgentUpdate(snapshot); const timelineSize = this.emitAgentTimelineSnapshot(snapshot); if (requestId) { @@ -5076,12 +5101,6 @@ export class Session { // Has content - abort any in-progress stream now this.createAbortController(); - // Wait for aborted stream to finish cleanup (save partial response) - if (this.currentStreamPromise) { - this.sessionLogger.debug("Waiting for aborted stream to finish cleanup"); - await this.currentStreamPromise; - } - if (result.debugRecordingPath) { this.emit({ type: "activity_log", @@ -5114,19 +5133,27 @@ export class Session { }); this.clearSpeechInProgress("transcription complete"); + this.setPhase("idle"); if (!this.isVoiceMode) { this.sessionLogger.debug( { requestId: result.requestId }, "Skipping voice agent processing because voice mode is disabled" ); - this.setPhase("idle"); return; } - this.setPhase("llm"); - this.currentStreamPromise = this.processVoiceTurn(result.text); - await this.currentStreamPromise; - this.setPhase("idle"); + const agentId = this.voiceModeAgentId; + if (!agentId) { + this.sessionLogger.warn( + { requestId: result.requestId }, + "Skipping voice agent processing because no agent is currently voice-enabled" + ); + return; + } + + // Route voice utterances through the same send path as regular text input: + // interrupt-if-running, record message, then start a new stream. + await this.handleSendAgentMessage(agentId, result.text); } private registerVoiceBridgeForAgent(agentId: string): void { @@ -5168,109 +5195,6 @@ export class Session { }); } - private buildVoiceAgentPrompt(userText: string): string { - const transcription = escapeXmlText(userText.trim()); - return [ - "", - transcription, - "", - ].join("\n"); - } - - private async processWithVoiceAgent( - agentId: string, - userText: string - ): Promise { - await this.ensureAgentLoaded(agentId); - await this.interruptAgentIfRunning(agentId); - - this.sessionLogger.info( - { - agentId, - userTextLength: userText.length, - userTextPreview: userText.slice(0, 160), - }, - "Starting voice agent turn" - ); - - const prompt = this.buildVoiceAgentPrompt(userText); - this.agentManager.recordUserMessage(agentId, userText); - - let sawSpeakToolCall = false; - const iterator = this.agentManager.streamAgent(agentId, prompt); - for await (const event of iterator) { - if (event.type === "turn_failed") { - throw new Error(event.error); - } - if (event.type === "timeline") { - if (event.item.type === "tool_call" && typeof event.item.name === "string") { - this.sessionLogger.info( - { - agentId, - toolName: event.item.name, - callId: event.item.callId, - }, - "Voice agent timeline tool call" - ); - if (event.item.name.toLowerCase().includes("speak")) { - sawSpeakToolCall = true; - } - } - } - if (event.type === "permission_requested") { - if (isVoicePermissionAllowed(event.request)) { - await this.agentManager.respondToPermission(agentId, event.request.id, { - behavior: "allow", - }); - } else { - this.sessionLogger.info( - { agentId, requestName: event.request.name, requestId: event.request.id }, - "Voice assistant left non-speak permission request for user decision" - ); - } - } - } - - if (!sawSpeakToolCall) { - this.sessionLogger.warn( - { agentId }, - "Voice agent turn completed without speak tool call; no audio playback emitted" - ); - } else { - this.sessionLogger.info( - { agentId }, - "Voice agent turn included speak tool call" - ); - } - } - - /** - * Process user message through LLM with streaming and tool execution - */ - private async processVoiceTurn(latestUserText?: string): Promise { - try { - if (!this.isVoiceMode) { - this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled"); - return; - } - const agentId = this.voiceModeAgentId; - if (!agentId) { - this.sessionLogger.warn( - "Ignoring processVoiceTurn call because no agent is currently voice-enabled" - ); - return; - } - const normalized = (latestUserText ?? "").trim(); - if (!normalized) { - return; - } - await this.processWithVoiceAgent(agentId, normalized); - } finally { - this.turnIndex++; - this.currentStreamPromise = null; - } - } - /** * Handle abort request from client */ @@ -5280,29 +5204,32 @@ export class Session { `Abort request, phase: ${this.processingPhase}` ); - if (this.processingPhase === "llm") { - // Already in LLM phase - abort and wait for cleanup - this.abortController.abort(); - this.sessionLogger.debug("Aborted LLM processing"); + this.abortController.abort(); + this.ttsManager.cancelPendingPlaybacks("abort request"); - // Wait for stream to finish saving partial response - if (this.currentStreamPromise) { - this.sessionLogger.debug("Waiting for stream cleanup after abort"); - await this.currentStreamPromise; + // Voice abort should always interrupt active agent output immediately. + if (this.isVoiceMode && this.voiceModeAgentId) { + try { + await this.interruptAgentIfRunning(this.voiceModeAgentId); + } catch (error) { + this.sessionLogger.warn( + { err: error, agentId: this.voiceModeAgentId }, + "Failed to interrupt active voice-mode agent on abort" + ); } + } - // Reset phase to idle - this.setPhase("idle"); - - // Clear any pending segments and timeouts - this.pendingAudioSegments = []; - this.clearBufferTimeout(); - } else if (this.processingPhase === "transcribing") { + if (this.processingPhase === "transcribing") { // Still in STT phase - we'll buffer the next audio this.sessionLogger.debug("Will buffer next audio (currently transcribing)"); // Phase stays as 'transcribing', handleAudioChunk will handle buffering + return; } - // If idle, nothing to do + + // Reset phase to idle and clear pending non-voice buffers. + this.setPhase("idle"); + this.pendingAudioSegments = []; + this.clearBufferTimeout(); } /** @@ -5313,7 +5240,7 @@ export class Session { } /** - * Mark speech detection start and abort any active playback/LLM + * Mark speech detection start and abort any active playback/agent run. */ private async handleVoiceSpeechStart(): Promise { if (this.speechInProgress) { @@ -5322,11 +5249,10 @@ export class Session { const chunkReceivedAt = Date.now(); const phaseBeforeAbort = this.processingPhase; - const hadActiveStream = Boolean(this.currentStreamPromise); + const hadActiveStream = this.hasActiveAgentRun(this.voiceModeAgentId); this.speechInProgress = true; - this.sessionLogger.debug("Voice speech detected – aborting playback and LLM"); - this.ttsManager.cancelPendingPlaybacks("voice speech detected"); + this.sessionLogger.debug("Voice speech detected – aborting playback and active agent run"); if (this.pendingAudioSegments.length > 0) { this.sessionLogger.debug( diff --git a/packages/server/src/server/speech/providers/local/config.ts b/packages/server/src/server/speech/providers/local/config.ts index 5399111ce..b70d0c6e2 100644 --- a/packages/server/src/server/speech/providers/local/config.ts +++ b/packages/server/src/server/speech/providers/local/config.ts @@ -144,6 +144,10 @@ export function resolveLocalSpeechConfig(params: { params.persisted.features?.voiceMode?.tts?.speed, }); + const resolvedVoiceTtsSpeakerId = + parsed.voiceLocalTtsSpeakerId ?? + (parsed.voiceLocalTtsModel === "kokoro-en-v0_19" ? 0 : undefined); + return { local: parsed.includeProviderConfig @@ -154,8 +158,8 @@ export function resolveLocalSpeechConfig(params: { dictationStt: parsed.dictationLocalSttModel, voiceStt: parsed.voiceLocalSttModel, voiceTts: parsed.voiceLocalTtsModel, - ...(parsed.voiceLocalTtsSpeakerId !== undefined - ? { voiceTtsSpeakerId: parsed.voiceLocalTtsSpeakerId } + ...(resolvedVoiceTtsSpeakerId !== undefined + ? { voiceTtsSpeakerId: resolvedVoiceTtsSpeakerId } : {}), ...(parsed.voiceLocalTtsSpeed !== undefined ? { voiceTtsSpeed: parsed.voiceLocalTtsSpeed } diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 0537c665d..3fcaf79d0 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -74,6 +74,7 @@ export const SHERPA_ONNX_MODEL_CATALOG = { requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"], description: "Kokoro TTS (higher quality; larger).", aliases: ["kokoro"], + defaultFor: "tts", }, "pocket-tts-onnx-int8": { kind: "tts", @@ -119,7 +120,6 @@ export const SHERPA_ONNX_MODEL_CATALOG = { ], description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).", aliases: ["pocket", "pocket-tts"], - defaultFor: "tts", }, } as const satisfies Record; diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts index c609be12d..495f93708 100644 --- a/packages/server/src/server/speech/speech-config-resolver.test.ts +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -36,12 +36,14 @@ describe("resolveSpeechConfig", () => { models: { dictationStt: "parakeet-tdt-0.6b-v2-int8", voiceStt: "parakeet-tdt-0.6b-v2-int8", - voiceTts: "pocket-tts-onnx-int8", + voiceTts: "kokoro-en-v0_19", + voiceTtsSpeakerId: 0, }, }); expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8"); expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8"); - expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8"); + expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19"); + expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0); }); test("resolves feature-scoped local model env vars", () => { @@ -122,6 +124,7 @@ describe("resolveSpeechConfig", () => { expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8"); expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8"); - expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8"); + expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19"); + expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0); }); }); diff --git a/packages/website/src/routes/docs/voice.tsx b/packages/website/src/routes/docs/voice.tsx index c7eb62bc3..4123df255 100644 --- a/packages/website/src/routes/docs/voice.tsx +++ b/packages/website/src/routes/docs/voice.tsx @@ -51,7 +51,7 @@ function VoiceDocs() {

Local Speech

Local speech defaults to model IDs parakeet-tdt-0.6b-v3-int8{' '} - (STT) and pocket-tts-onnx-int8 (TTS). + (STT) and kokoro-en-v0_19 (TTS, speaker 0 / voice 00).

Missing models are downloaded at daemon startup into{' '} @@ -66,7 +66,7 @@ function VoiceDocs() { "voiceMode": { "llm": { "provider": "claude", "model": "haiku" }, "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8" }, - "tts": { "provider": "local", "model": "pocket-tts-onnx-int8" } + "tts": { "provider": "local", "model": "kokoro-en-v0_19", "speakerId": 0 } } }, "providers": {