Update files

packages/app/src/components/volume-meter.tsx

@@ -24,7 +24,6 @@ interface VolumeMeterProps {
 export function VolumeMeter({
   volume,
   isMuted = false,
-  isDetecting = false,
   isSpeaking = false,
   orientation = "vertical",
   variant = "default",
@@ -146,9 +145,9 @@ export function VolumeMeter({
 
   // Create animated styles unconditionally at top level
   const line1Style = useAnimatedStyle(() => {
-    const isActive = isDetecting || isSpeaking;
+    const isActive = isSpeaking;
     const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
-    const volumeBoost = isMuted ? 0 : volume * 0.3;
+    const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
     return {
       height: line1Height.value * (isMuted || volume > 0.001 ? 1 : line1Pulse.value),
       opacity: baseOpacity + volumeBoost,
@@ -156,9 +155,9 @@ export function VolumeMeter({
   });
 
   const line2Style = useAnimatedStyle(() => {
-    const isActive = isDetecting || isSpeaking;
+    const isActive = isSpeaking;
     const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
-    const volumeBoost = isMuted ? 0 : volume * 0.3;
+    const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
     return {
       height: line2Height.value * (isMuted || volume > 0.001 ? 1 : line2Pulse.value),
       opacity: baseOpacity + volumeBoost,
@@ -166,9 +165,9 @@ export function VolumeMeter({
   });
 
   const line3Style = useAnimatedStyle(() => {
-    const isActive = isDetecting || isSpeaking;
+    const isActive = isSpeaking;
     const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
-    const volumeBoost = isMuted ? 0 : volume * 0.3;
+    const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
     return {
       height: line3Height.value * (isMuted || volume > 0.001 ? 1 : line3Pulse.value),
       opacity: baseOpacity + volumeBoost,

packages/app/src/contexts/voice-context.tsx

@@ -63,10 +63,15 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
   const lastVoiceModeSyncedClientRef = useRef<SessionState["client"] | null>(null);
   const voiceTransportReadyRef = useRef(false);
   const voiceResyncInFlightRef = useRef(false);
+  const silenceGraceStartMsRef = useRef<number | null>(null);
+  const vadStateRef = useRef<{ isDetecting: boolean; isSpeaking: boolean }>({
+    isDetecting: false,
+    isSpeaking: false,
+  });
 
   const realtimeAudio = useSpeechmaticsAudio({
     onSpeechStart: () => {
-      console.log("[Voice] Speech detected");
+      console.log("[Voice] Segment started (speech confirmed)");
       // Stop audio playback if playing
       const session = realtimeSessionRef.current;
       const sessionAudioPlayer = session?.audioPlayer ?? null;
@@ -93,7 +98,16 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
       }
     },
     onSpeechEnd: () => {
-      console.log("[Voice] Speech ended");
+      const silenceMs =
+        silenceGraceStartMsRef.current === null
+          ? null
+          : Date.now() - silenceGraceStartMsRef.current;
+      if (silenceMs === null) {
+        console.log("[Voice] Segment finalized");
+      } else {
+        console.log("[Voice] Segment finalized", { silenceMs });
+      }
+      silenceGraceStartMsRef.current = null;
     },
     onAudioSegment: ({ audioData, isLast }) => {
       if (!voiceTransportReadyRef.current) {
@@ -134,6 +148,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
       }
     },
     volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
+    confirmedDropGracePeriod: REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
     silenceDuration: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
     speechConfirmationDuration: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
     detectionGracePeriod: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
@@ -143,6 +158,40 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
     realtimeSessionRef.current = activeSession;
   }, [activeSession]);
 
+  useEffect(() => {
+    const next = {
+      isDetecting: realtimeAudio.isDetecting,
+      isSpeaking: realtimeAudio.isSpeaking,
+    };
+    const prev = vadStateRef.current;
+
+    if (!isVoiceMode) {
+      silenceGraceStartMsRef.current = null;
+      vadStateRef.current = next;
+      return;
+    }
+
+    // Confirmed speech dropped below threshold: grace window starts.
+    if (prev.isSpeaking && !next.isSpeaking && next.isDetecting) {
+      silenceGraceStartMsRef.current = Date.now();
+      console.log("[Voice] Grace started (speech dropped below threshold)");
+    }
+
+    // User resumed speaking before grace timeout elapsed.
+    if (!prev.isSpeaking && next.isSpeaking && silenceGraceStartMsRef.current !== null) {
+      const resumedAfterMs = Date.now() - silenceGraceStartMsRef.current;
+      console.log("[Voice] Speech resumed during grace", { resumedAfterMs });
+      silenceGraceStartMsRef.current = null;
+    }
+
+    // Fully idle (neither detecting nor speaking).
+    if (!next.isDetecting && !next.isSpeaking) {
+      silenceGraceStartMsRef.current = null;
+    }
+
+    vadStateRef.current = next;
+  }, [isVoiceMode, realtimeAudio.isDetecting, realtimeAudio.isSpeaking]);
+
   useEffect(() => {
     const connected = activeSession?.connection.isConnected ?? false;
     const client = activeSession?.client ?? null;

packages/app/src/hooks/use-speechmatics-audio.native.ts

@@ -19,6 +19,8 @@ export interface SpeechmaticsAudioConfig {
   /** When true, stream microphone PCM continuously without VAD gating. */
   enableContinuousStreaming?: boolean;
   volumeThreshold: number; // Volume threshold for speech detection (0-1)
+  /** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
+  confirmedDropGracePeriod?: number;
   silenceDuration: number; // ms of silence before ending segment
   speechConfirmationDuration: number; // ms of sustained speech before confirming
   detectionGracePeriod: number; // ms grace period for volume dips during detection
@@ -83,6 +85,7 @@ export function useSpeechmaticsAudio(
       {
         enableContinuousStreaming,
         volumeThreshold: config.volumeThreshold,
+        confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
         silenceDurationMs: config.silenceDuration,
         speechConfirmationMs: config.speechConfirmationDuration,
         detectionGracePeriodMs: config.detectionGracePeriod,
@@ -101,6 +104,7 @@ export function useSpeechmaticsAudio(
     segmenterRef.current?.updateConfig({
       enableContinuousStreaming,
       volumeThreshold: config.volumeThreshold,
+      confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
       silenceDurationMs: config.silenceDuration,
       speechConfirmationMs: config.speechConfirmationDuration,
       detectionGracePeriodMs: config.detectionGracePeriod,
@@ -108,6 +112,7 @@ export function useSpeechmaticsAudio(
   }, [
     enableContinuousStreaming,
     config.volumeThreshold,
+    config.confirmedDropGracePeriod,
     config.silenceDuration,
     config.speechConfirmationDuration,
     config.detectionGracePeriod,

packages/app/src/hooks/use-speechmatics-audio.web.ts

@@ -10,6 +10,8 @@ export interface SpeechmaticsAudioConfig {
   /** When true, stream microphone PCM continuously without VAD gating. */
   enableContinuousStreaming?: boolean;
   volumeThreshold: number; // 0-1
+  /** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
+  confirmedDropGracePeriod?: number;
   silenceDuration: number; // ms of silence before ending segment
   speechConfirmationDuration: number; // ms of sustained speech before confirming
   detectionGracePeriod: number; // ms grace period for volume dips during detection
@@ -100,6 +102,7 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
       {
         enableContinuousStreaming,
         volumeThreshold: config.volumeThreshold,
+        confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
         silenceDurationMs: config.silenceDuration,
         speechConfirmationMs: config.speechConfirmationDuration,
         detectionGracePeriodMs: config.detectionGracePeriod,
@@ -118,6 +121,7 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
     segmenterRef.current?.updateConfig({
       enableContinuousStreaming,
       volumeThreshold: config.volumeThreshold,
+      confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
       silenceDurationMs: config.silenceDuration,
       speechConfirmationMs: config.speechConfirmationDuration,
       detectionGracePeriodMs: config.detectionGracePeriod,
@@ -125,6 +129,7 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
   }, [
     enableContinuousStreaming,
     config.volumeThreshold,
+    config.confirmedDropGracePeriod,
     config.silenceDuration,
     config.speechConfirmationDuration,
     config.detectionGracePeriod,
@@ -151,6 +156,21 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
     isMutedRef.current = isMuted;
   }, [isMuted]);
 
+  const vadConfigRef = useRef({
+    volumeThreshold: config.volumeThreshold,
+  });
+  useEffect(() => {
+    vadConfigRef.current = {
+      volumeThreshold: config.volumeThreshold,
+    };
+  }, [config.volumeThreshold]);
+
+  const vadLogRef = useRef({
+    lastLogMs: 0,
+    lastSpeaking: false,
+    lastDetecting: false,
+  });
+
   // Update segment duration timer
   useEffect(() => {
     if (!isDetecting && !isSpeaking) {
@@ -288,6 +308,30 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
 
         const nowMs = Date.now();
         segmenterRef.current?.pushVolumeLevel(normalized, nowMs);
+        const segmenter = segmenterRef.current;
+        if (segmenter) {
+          const speakingNow = segmenter.getIsSpeaking();
+          const detectingNow = segmenter.getIsDetecting();
+          const shouldLog =
+            nowMs - vadLogRef.current.lastLogMs >= 150 ||
+            speakingNow !== vadLogRef.current.lastSpeaking ||
+            detectingNow !== vadLogRef.current.lastDetecting;
+          if (shouldLog) {
+            const threshold = vadConfigRef.current.volumeThreshold;
+            const releaseThreshold = segmenter.getVolumeReleaseThreshold();
+            console.log("[Voice][Web][VAD] level", {
+              volume: Number(normalized.toFixed(3)),
+              threshold: Number(threshold.toFixed(3)),
+              releaseThreshold: Number(releaseThreshold.toFixed(3)),
+              confirmedDropGraceMs: config.confirmedDropGracePeriod ?? 250,
+              isSpeaking: speakingNow,
+              isDetecting: detectingNow,
+            });
+            vadLogRef.current.lastLogMs = nowMs;
+            vadLogRef.current.lastSpeaking = speakingNow;
+            vadLogRef.current.lastDetecting = detectingNow;
+          }
+        }
 
         const pcm16 = resampleToPcm16(input, context.sampleRate, 16000);
         const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength);
@@ -347,4 +391,3 @@ export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): Speechmat
     [start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration]
   );
 }
-

packages/app/src/screens/agent/agent-ready-screen.tsx

@@ -553,7 +553,10 @@ function AgentScreenContent({
     if (!isConnected) {
       return;
     }
-    if (!needsAuthoritativeSync) {
+    // On native clients, daemon stream forwarding is focused-agent only, so switching
+    // agents can leave timeline gaps unless we explicitly request a snapshot.
+    const shouldSyncOnEntry = needsAuthoritativeSync || Platform.OS !== "web";
+    if (!shouldSyncOnEntry) {
       return;
     }
 
@@ -563,7 +566,12 @@ function AgentScreenContent({
         error,
       });
     });
-  }, [resolvedAgentId, ensureAgentIsInitialized, isConnected, needsAuthoritativeSync]);
+  }, [
+    resolvedAgentId,
+    ensureAgentIsInitialized,
+    isConnected,
+    needsAuthoritativeSync,
+  ]);
 
   useEffect(() => {
     // Clear stale resolution state when route target changes.

packages/app/src/screens/agent/draft-agent-screen.tsx

@@ -400,6 +400,7 @@ export function DraftAgentScreen({
   const checkoutQueryError =
     checkoutStatusQuery.error instanceof Error ? checkoutStatusQuery.error.message : null;
   const checkoutPayloadError = checkout?.error ? checkout.error.message : null;
+  const isGitDirectory = checkoutStatusQuery.isSuccess && checkout?.isGit === true;
 
   const isNonGitDirectory =
     Boolean(trimmedWorkingDir) &&
@@ -430,7 +431,7 @@ export function DraftAgentScreen({
   const isCreateWorktree = worktreeMode === "create";
   const isAttachWorktree = worktreeMode === "attach";
 
-  const worktreeListRoot = checkout?.repoRoot ?? trimmedWorkingDir;
+  const worktreeListRoot = checkout?.isGit ? checkout.repoRoot : "";
   const worktreeListQuery = useQuery({
     queryKey: ["paseoWorktreeList", selectedServerId, worktreeListRoot],
     queryFn: async () => {
@@ -448,7 +449,8 @@ export function DraftAgentScreen({
       return payload.worktrees ?? [];
     },
     enabled:
-      Boolean(worktreeListRoot || trimmedWorkingDir) &&
+      isGitDirectory &&
+      Boolean(worktreeListRoot) &&
       !repoAvailabilityError &&
       Boolean(sessionClient) &&
       isConnected &&
@@ -551,6 +553,7 @@ export function DraftAgentScreen({
     },
     enabled:
       isCreateWorktree &&
+      isGitDirectory &&
       !isNonGitDirectory &&
       Boolean(baseBranch) &&
       Boolean(trimmedWorkingDir) &&

packages/app/src/voice/realtime-voice-config.ts

@@ -1,7 +1,12 @@
 export const REALTIME_VOICE_VAD_CONFIG = {
-  volumeThreshold: 0.18,
+  // Headset mics often report lower input volume than built-in mics.
-  silenceDurationMs: 2200,
+  // Keep threshold low enough to avoid requiring loud speech.
+  volumeThreshold: 0.12,
+  // Short staggered grace before we visually/VAD-deactivate speaking.
+  // Prevents fade-out between normal intra-word pauses.
+  confirmedDropGracePeriodMs: 1000,
+  // Keep turns open longer so brief "thinking pauses" don't end speech too early.
+  silenceDurationMs: 2000,
   speechConfirmationMs: 120,
   detectionGracePeriodMs: 700,
 } as const;
-

packages/app/src/voice/speech-segmenter.test.ts

@@ -68,20 +68,24 @@ describe("SpeechSegmenter", () => {
     segmenter.pushVolumeLevel(0.5, t0);
     segmenter.pushPcmChunk(mkPcmBytes(50));
     expect(detectingChanges).toEqual([true]);
+    expect(speakingChanges).toEqual([true]);
     expect(onSpeechStart).not.toHaveBeenCalled();
 
     // Confirm speech.
     segmenter.pushVolumeLevel(0.5, t0 + 300);
     segmenter.pushPcmChunk(mkPcmBytes(50));
     expect(onSpeechStart).toHaveBeenCalledTimes(1);
+    expect(detectingChanges).toEqual([true, false]);
     expect(speakingChanges).toEqual([true]);
 
     // End speech after silence.
     segmenter.pushVolumeLevel(0.0, t0 + 301);
     segmenter.pushPcmChunk(mkPcmBytes(50));
     segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000);
+    segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000 + 2000);
 
     expect(onSpeechEnd).toHaveBeenCalledTimes(1);
+    expect(detectingChanges).toEqual([true, false, true, false]);
     expect(speakingChanges).toEqual([true, false]);
     expect(onAudioSegment).toHaveBeenCalled();
     const lastCall = onAudioSegment.mock.calls.at(-1)![0];
@@ -121,4 +125,148 @@ describe("SpeechSegmenter", () => {
     expect(detectingChanges).toContain(true);
     expect(onSpeechStart).toHaveBeenCalledTimes(1);
   });
+
+  it("keeps utterance open after fade-out and resumes speaking during silence grace", () => {
+    const onAudioSegment = vi.fn();
+    const onSpeechStart = vi.fn();
+    const onSpeechEnd = vi.fn();
+    const speakingChanges: boolean[] = [];
+
+    const segmenter = new SpeechSegmenter(
+      {
+        enableContinuousStreaming: false,
+        volumeThreshold: 0.2,
+        silenceDurationMs: 1500,
+        speechConfirmationMs: 120,
+        detectionGracePeriodMs: 400,
+        minChunkDurationMs: 1000, // keep everything buffered until final flush
+        pcmSampleRate: 1000,
+      },
+      {
+        onAudioSegment,
+        onSpeechStart,
+        onSpeechEnd,
+        onSpeakingChange: (v) => speakingChanges.push(v),
+      }
+    );
+
+    const t0 = 30_000;
+
+    // Start + confirm speaking.
+    segmenter.pushVolumeLevel(0.6, t0);
+    segmenter.pushPcmChunk(mkPcmBytes(200));
+    segmenter.pushVolumeLevel(0.6, t0 + 130);
+    segmenter.pushPcmChunk(mkPcmBytes(200));
+    expect(onSpeechStart).toHaveBeenCalledTimes(1);
+
+    // Drop below threshold: meter should fade, but turn remains open.
+    segmenter.pushVolumeLevel(0.0, t0 + 200);
+    segmenter.pushPcmChunk(mkPcmBytes(200));
+    segmenter.pushVolumeLevel(0.0, t0 + 500);
+    segmenter.pushPcmChunk(mkPcmBytes(200));
+
+    // Speak again before silence timeout expires: continue same utterance.
+    segmenter.pushVolumeLevel(0.6, t0 + 900);
+    segmenter.pushPcmChunk(mkPcmBytes(200));
+    expect(onSpeechStart).toHaveBeenCalledTimes(1);
+    expect(onSpeechEnd).not.toHaveBeenCalled();
+
+    // End for real.
+    segmenter.pushVolumeLevel(0.0, t0 + 1000);
+    segmenter.pushPcmChunk(mkPcmBytes(200));
+    segmenter.pushVolumeLevel(0.0, t0 + 2600);
+    segmenter.pushVolumeLevel(0.0, t0 + 4200);
+
+    expect(speakingChanges).toEqual([true, false, true, false]);
+    expect(onSpeechEnd).toHaveBeenCalledTimes(1);
+    const finalCalls = onAudioSegment.mock.calls.filter((call) => call[0].isLast);
+    expect(finalCalls).toHaveLength(1);
+  });
+
+  it("keeps confirmed speech active above release threshold even when below start threshold", () => {
+    const onSpeechStart = vi.fn();
+    const onSpeechEnd = vi.fn();
+    const speakingChanges: boolean[] = [];
+
+    const segmenter = new SpeechSegmenter(
+      {
+        enableContinuousStreaming: false,
+        volumeThreshold: 0.3,
+        volumeReleaseThreshold: 0.1,
+        silenceDurationMs: 1200,
+        speechConfirmationMs: 120,
+        detectionGracePeriodMs: 200,
+        minChunkDurationMs: 1000,
+        pcmSampleRate: 1000,
+      },
+      {
+        onSpeechStart,
+        onSpeechEnd,
+        onSpeakingChange: (v) => speakingChanges.push(v),
+      }
+    );
+
+    const t0 = 40_000;
+
+    // Confirm speech.
+    segmenter.pushVolumeLevel(0.4, t0);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    segmenter.pushVolumeLevel(0.4, t0 + 130);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    expect(onSpeechStart).toHaveBeenCalledTimes(1);
+
+    // Stay between start and release thresholds for longer than silence timeout:
+    // should still be considered speaking.
+    segmenter.pushVolumeLevel(0.15, t0 + 400);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    segmenter.pushVolumeLevel(0.15, t0 + 1000);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    segmenter.pushVolumeLevel(0.15, t0 + 2000);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+
+    expect(onSpeechEnd).not.toHaveBeenCalled();
+    expect(speakingChanges.at(-1)).toBe(true);
+
+    // Now actually drop below release threshold and allow silence timeout.
+    segmenter.pushVolumeLevel(0.0, t0 + 2001);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    segmenter.pushVolumeLevel(0.0, t0 + 3405);
+    segmenter.pushVolumeLevel(0.0, t0 + 4706);
+
+    expect(onSpeechEnd).toHaveBeenCalledTimes(1);
+    expect(speakingChanges.at(-1)).toBe(false);
+  });
+
+  it("ignores brief confirmed dips below release threshold without flip-flopping speaking", () => {
+    const speakingChanges: boolean[] = [];
+
+    const segmenter = new SpeechSegmenter(
+      {
+        enableContinuousStreaming: false,
+        volumeThreshold: 0.2,
+        volumeReleaseThreshold: 0.08,
+        confirmedDropGracePeriodMs: 250,
+        silenceDurationMs: 1500,
+        speechConfirmationMs: 120,
+        detectionGracePeriodMs: 200,
+        minChunkDurationMs: 1000,
+        pcmSampleRate: 1000,
+      },
+      {
+        onSpeakingChange: (v) => speakingChanges.push(v),
+      }
+    );
+
+    const t0 = 50_000;
+    segmenter.pushVolumeLevel(0.4, t0);
+    segmenter.pushVolumeLevel(0.4, t0 + 130); // confirm speaking
+
+    // Two short dips below release threshold; both shorter than debounce.
+    segmenter.pushVolumeLevel(0.05, t0 + 300);
+    segmenter.pushVolumeLevel(0.09, t0 + 500);
+    segmenter.pushVolumeLevel(0.05, t0 + 620);
+    segmenter.pushVolumeLevel(0.1, t0 + 760);
+
+    expect(speakingChanges).toEqual([true]);
+  });
 });

packages/app/src/voice/speech-segmenter.ts

@@ -5,6 +5,8 @@ export type SpeechSegment = { audioData: string; isLast: boolean };
 export type SpeechSegmenterConfig = {
   enableContinuousStreaming: boolean;
   volumeThreshold: number; // 0..1
+  volumeReleaseThreshold?: number; // 0..1; lower threshold used to keep active speech from dropping out
+  confirmedDropGracePeriodMs?: number; // ms dip debounce before leaving confirmed speaking
   silenceDurationMs: number;
   speechConfirmationMs: number;
   detectionGracePeriodMs: number;
@@ -58,6 +60,7 @@ export class SpeechSegmenter {
 
   private speechDetectionStartMs: number | null = null;
   private detectionSilenceStartMs: number | null = null;
+  private confirmedDropStartMs: number | null = null;
   private silenceStartMs: number | null = null;
   private isSpeaking = false;
   private isDetecting = false;
@@ -67,8 +70,14 @@ export class SpeechSegmenter {
   private minChunkBytes: number;
 
   constructor(config: SpeechSegmenterConfig, callbacks: SpeechSegmenterCallbacks = {}) {
+    const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold(
+      config.volumeThreshold,
+      config.volumeReleaseThreshold
+    );
     this.config = {
       ...config,
+      volumeReleaseThreshold,
+      confirmedDropGracePeriodMs: config.confirmedDropGracePeriodMs ?? 250,
       minChunkDurationMs: config.minChunkDurationMs ?? 1000,
       pcmSampleRate: config.pcmSampleRate ?? 16000,
       pcmChannels: config.pcmChannels ?? 1,
@@ -81,8 +90,14 @@ export class SpeechSegmenter {
   }
 
   updateConfig(next: SpeechSegmenterConfig): void {
+    const volumeReleaseThreshold = this.resolveVolumeReleaseThreshold(
+      next.volumeThreshold,
+      next.volumeReleaseThreshold
+    );
     this.config = {
       ...next,
+      volumeReleaseThreshold,
+      confirmedDropGracePeriodMs: next.confirmedDropGracePeriodMs ?? this.config.confirmedDropGracePeriodMs,
       minChunkDurationMs: next.minChunkDurationMs ?? this.config.minChunkDurationMs,
       pcmSampleRate: next.pcmSampleRate ?? this.config.pcmSampleRate,
       pcmChannels: next.pcmChannels ?? this.config.pcmChannels,
@@ -107,11 +122,16 @@ export class SpeechSegmenter {
     return this.isDetecting;
   }
 
+  getVolumeReleaseThreshold(): number {
+    return this.config.volumeReleaseThreshold;
+  }
+
   reset(): void {
     this.audioBuffer = [];
     this.bufferedBytes = 0;
     this.speechDetectionStartMs = null;
     this.detectionSilenceStartMs = null;
+    this.confirmedDropStartMs = null;
     this.silenceStartMs = null;
     this.isSpeaking = false;
     this.isDetecting = false;
@@ -170,79 +190,103 @@ export class SpeechSegmenter {
       return;
     }
 
-    const speechDetected = volume > this.config.volumeThreshold;
+    const speechDetectedForStart = volume > this.config.volumeThreshold;
+    const speechDetectedForContinue = volume > this.config.volumeReleaseThreshold;
 
-    if (speechDetected && !this.isSpeaking && !this.speechConfirmed) {
+    if (!this.speechConfirmed) {
-      // Initial detection phase.
+      if (speechDetectedForStart) {
-      if (this.speechDetectionStartMs === null) {
+        // Initial detection phase.
-        this.speechDetectionStartMs = nowMs;
+        this.confirmedDropStartMs = null;
-        this.detectionSilenceStartMs = null;
+        if (this.speechDetectionStartMs === null) {
-        this.audioBuffer = [];
+          this.speechDetectionStartMs = nowMs;
-        this.bufferedBytes = 0;
+          this.detectionSilenceStartMs = null;
-        this.setDetecting(true);
+          this.audioBuffer = [];
-        return;
+          this.bufferedBytes = 0;
-      }
+          this.setDetecting(true);
-
+          this.setSpeaking(true);
-      // Volume is back above threshold - reset grace period.
+          return;
-      this.detectionSilenceStartMs = null;
-
-      const speechDuration = nowMs - this.speechDetectionStartMs;
-      if (speechDuration >= this.config.speechConfirmationMs) {
-        this.speechConfirmed = true;
-        this.silenceStartMs = null;
-        this.setDetecting(false);
-        this.setSpeaking(true);
-        this.callbacks.onSpeechStart?.();
-      }
-      return;
-    }
-
-    if (speechDetected && this.isSpeaking && this.speechConfirmed) {
-      // Continuing confirmed speech.
-      this.silenceStartMs = null;
-      return;
-    }
-
-    if (!speechDetected && !this.speechConfirmed && this.speechDetectionStartMs !== null) {
-      // Volume dropped during detection phase - apply grace period.
-      if (this.detectionSilenceStartMs === null) {
-        this.detectionSilenceStartMs = nowMs;
-        return;
-      }
-
-      const graceDuration = nowMs - this.detectionSilenceStartMs;
-      if (graceDuration >= this.config.detectionGracePeriodMs) {
-        // Cancel detection.
-        this.speechDetectionStartMs = null;
-        this.detectionSilenceStartMs = null;
-        this.audioBuffer = [];
-        this.bufferedBytes = 0;
-        this.setDetecting(false);
-      }
-      return;
-    }
-
-    if (!speechDetected && this.isSpeaking && this.speechConfirmed) {
-      // Potential speech END.
-      if (this.silenceStartMs === null) {
-        this.silenceStartMs = nowMs;
-        return;
-      }
-
-      const silenceDuration = nowMs - this.silenceStartMs;
-      if (silenceDuration >= this.config.silenceDurationMs) {
-        // Speech END confirmed.
-        this.speechConfirmed = false;
-        this.speechDetectionStartMs = null;
-        this.silenceStartMs = null;
-        this.setSpeaking(false);
-        this.callbacks.onSpeechEnd?.();
-        if (this.audioBuffer.length === 0) {
-          // Important: downstream expects an explicit segment boundary marker.
-          this.callbacks.onAudioSegment?.({ audioData: "", isLast: true });
-        } else {
-          this.flush(true);
         }
+
+        // Volume is back above threshold - reset grace period.
+        this.detectionSilenceStartMs = null;
+        this.setSpeaking(true);
+
+        const speechDuration = nowMs - this.speechDetectionStartMs;
+        if (speechDuration >= this.config.speechConfirmationMs) {
+          this.speechConfirmed = true;
+          this.silenceStartMs = null;
+          this.setDetecting(false);
+          this.callbacks.onSpeechStart?.();
+        }
+        return;
+      }
+
+      if (this.speechDetectionStartMs !== null) {
+        // Volume dropped during detection phase - apply grace period.
+        if (this.detectionSilenceStartMs === null) {
+          this.detectionSilenceStartMs = nowMs;
+          this.setSpeaking(false);
+          return;
+        }
+
+        const graceDuration = nowMs - this.detectionSilenceStartMs;
+        if (graceDuration >= this.config.detectionGracePeriodMs) {
+          // Cancel detection.
+          this.speechDetectionStartMs = null;
+          this.detectionSilenceStartMs = null;
+          this.confirmedDropStartMs = null;
+          this.audioBuffer = [];
+          this.bufferedBytes = 0;
+          this.setDetecting(false);
+          this.setSpeaking(false);
+        }
+      }
+      return;
+    }
+
+    if (speechDetectedForContinue) {
+      // Continuing confirmed speech (or resuming during silence grace).
+      this.confirmedDropStartMs = null;
+      this.silenceStartMs = null;
+      this.setDetecting(false);
+      this.setSpeaking(true);
+      return;
+    }
+
+    if (this.confirmedDropStartMs === null) {
+      this.confirmedDropStartMs = nowMs;
+      return;
+    }
+
+    const confirmedDropDuration = nowMs - this.confirmedDropStartMs;
+    if (confirmedDropDuration < this.config.confirmedDropGracePeriodMs) {
+      return;
+    }
+
+    // Potential speech END while utterance remains open.
+    if (this.silenceStartMs === null) {
+      this.silenceStartMs = nowMs;
+      this.setDetecting(true);
+      this.setSpeaking(false);
+      return;
+    }
+
+    const silenceDuration = nowMs - this.silenceStartMs;
+    if (silenceDuration >= this.config.silenceDurationMs) {
+      // Speech END confirmed.
+      this.speechConfirmed = false;
+      this.speechDetectionStartMs = null;
+      this.detectionSilenceStartMs = null;
+      this.confirmedDropStartMs = null;
+      this.silenceStartMs = null;
+      this.setDetecting(false);
+      this.setSpeaking(false);
+      this.callbacks.onSpeechEnd?.();
+      if (this.audioBuffer.length === 0) {
+        // Important: downstream expects an explicit segment boundary marker.
+        this.callbacks.onAudioSegment?.({ audioData: "", isLast: true });
+      } else {
+        this.flush(true);
       }
     }
   }
@@ -282,4 +326,13 @@ export class SpeechSegmenter {
       1000;
     this.minChunkBytes = Math.round(this.pcmBytesPerMs * this.config.minChunkDurationMs);
   }
+
+  private resolveVolumeReleaseThreshold(
+    volumeThreshold: number,
+    configured?: number
+  ): number {
+    const defaultReleaseThreshold = volumeThreshold * 0.4;
+    const raw = configured ?? defaultReleaseThreshold;
+    return Math.max(0, Math.min(raw, volumeThreshold));
+  }
 }

packages/server/package.json

@@ -42,6 +42,7 @@
     "generate:config-schema": "tsx scripts/generate-config-schema.ts",
     "speech:models": "tsx scripts/list-speech-models.ts",
     "speech:download": "tsx scripts/download-speech-models.ts",
+    "speech:tts:matrix": "tsx scripts/generate-sherpa-tts-matrix.ts",
     "speech:transcribe:local": "tsx scripts/transcribe-local-wav.ts",
     "test": "vitest run",
     "test:watch": "vitest",

packages/server/scripts/generate-sherpa-tts-matrix.ts

@@ -0,0 +1,433 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+import { createRootLogger } from "../src/server/logger.js";
+import { resolvePaseoHome } from "../src/server/paseo-home.js";
+import { float32ToPcm16le, parsePcmRateFromFormat } from "../src/server/speech/audio.js";
+import {
+  ensureLocalSpeechModels,
+  getLocalSpeechModelDir,
+  type LocalTtsModelId,
+} from "../src/server/speech/providers/local/models.js";
+import { PocketTtsOnnxTTS } from "../src/server/speech/providers/local/pocket/pocket-tts-onnx.js";
+import {
+  LOCAL_TTS_MODEL_IDS,
+  LocalTtsModelIdSchema,
+} from "../src/server/speech/providers/local/sherpa/model-catalog.js";
+import { loadSherpaOnnxNode } from "../src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.js";
+
+const DEFAULT_SAMPLE_TEXT =
+  "Paseo keeps your coding agents in your pocket, with fast local speech and reliable remote control.";
+
+type ScriptOptions = {
+  modelsDir: string;
+  outputDir: string;
+  text: string;
+  speed: number;
+  modelIds: LocalTtsModelId[];
+};
+
+type GeneratedSample = {
+  modelId: LocalTtsModelId;
+  voiceLabel: string;
+  speakerId: number | null;
+  sampleRate: number;
+  durationSeconds: number;
+  relativePath: string;
+};
+
+function usage(): string {
+  return [
+    "Generate a local Sherpa/Pocket TTS sample matrix.",
+    "",
+    "Options:",
+    "  --models-dir <dir>   Directory for downloaded local speech models",
+    "  --output-dir <dir>   Directory where WAV samples are written",
+    "  --model <id>         TTS model to include (repeatable)",
+    "  --text <sentence>    Sample sentence used for every generated file",
+    "  --speed <number>     TTS speed multiplier for Sherpa models (default: 1.0)",
+    "  --help               Show this help",
+    "",
+    `Known TTS model IDs: ${LOCAL_TTS_MODEL_IDS.join(", ")}`,
+  ].join("\n");
+}
+
+function parsePositiveNumber(raw: string, flag: string): number {
+  const parsed = Number.parseFloat(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    throw new Error(`${flag} must be a positive number (received: ${raw})`);
+  }
+  return parsed;
+}
+
+function parseArgs(argv: string[]): ScriptOptions {
+  const paseoHome = resolvePaseoHome();
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+
+  let modelsDir = process.env.PASEO_LOCAL_MODELS_DIR ?? `${paseoHome}/models/local-speech`;
+  let outputDir = path.resolve(process.cwd(), ".debug", "sherpa-tts-matrix", timestamp);
+  let text = DEFAULT_SAMPLE_TEXT;
+  let speed = parsePositiveNumber(process.env.PASEO_VOICE_LOCAL_TTS_SPEED ?? "1.0", "--speed");
+  const requestedModelIds: LocalTtsModelId[] = [];
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+
+    if (arg === "--help") {
+      // eslint-disable-next-line no-console
+      console.log(usage());
+      process.exit(0);
+    }
+
+    if (arg === "--models-dir") {
+      modelsDir = path.resolve(argv[i + 1] ?? modelsDir);
+      i += 1;
+      continue;
+    }
+
+    if (arg === "--output-dir") {
+      outputDir = path.resolve(argv[i + 1] ?? outputDir);
+      i += 1;
+      continue;
+    }
+
+    if (arg === "--text") {
+      text = argv[i + 1] ?? text;
+      i += 1;
+      continue;
+    }
+
+    if (arg === "--speed") {
+      const raw = argv[i + 1];
+      if (!raw) {
+        throw new Error("--speed requires a value");
+      }
+      speed = parsePositiveNumber(raw, "--speed");
+      i += 1;
+      continue;
+    }
+
+    if (arg === "--model") {
+      const raw = argv[i + 1];
+      if (!raw) {
+        throw new Error("--model requires a value");
+      }
+      requestedModelIds.push(LocalTtsModelIdSchema.parse(raw));
+      i += 1;
+      continue;
+    }
+
+    throw new Error(`Unknown argument: ${arg}`);
+  }
+
+  const trimmed = text.trim();
+  if (!trimmed) {
+    throw new Error("Sample text cannot be empty");
+  }
+
+  const modelIds =
+    requestedModelIds.length > 0
+      ? Array.from(new Set(requestedModelIds))
+      : [...LOCAL_TTS_MODEL_IDS];
+
+  return {
+    modelsDir,
+    outputDir,
+    text: trimmed,
+    speed,
+    modelIds,
+  };
+}
+
+function pcm16MonoToWavBuffer(pcm16: Buffer, sampleRate: number): Buffer {
+  const channels = 1;
+  const bitsPerSample = 16;
+  const headerSize = 44;
+  const wav = Buffer.alloc(headerSize + pcm16.length);
+  const byteRate = (sampleRate * channels * bitsPerSample) / 8;
+  const blockAlign = (channels * bitsPerSample) / 8;
+
+  wav.write("RIFF", 0);
+  wav.writeUInt32LE(36 + pcm16.length, 4);
+  wav.write("WAVE", 8);
+  wav.write("fmt ", 12);
+  wav.writeUInt32LE(16, 16);
+  wav.writeUInt16LE(1, 20);
+  wav.writeUInt16LE(channels, 22);
+  wav.writeUInt32LE(sampleRate, 24);
+  wav.writeUInt32LE(byteRate, 28);
+  wav.writeUInt16LE(blockAlign, 32);
+  wav.writeUInt16LE(bitsPerSample, 34);
+  wav.write("data", 36);
+  wav.writeUInt32LE(pcm16.length, 40);
+  pcm16.copy(wav, 44);
+
+  return wav;
+}
+
+function sanitizeFilePart(input: string): string {
+  return input.replace(/[^a-zA-Z0-9._-]/g, "-");
+}
+
+function toFloatSamples(audio: unknown): Float32Array {
+  const raw = (audio as { samples?: unknown })?.samples;
+  if (raw instanceof Float32Array) {
+    return raw;
+  }
+  if (Array.isArray(raw)) {
+    return Float32Array.from(raw as number[]);
+  }
+  throw new Error("Unexpected Sherpa TTS output: missing Float32 samples");
+}
+
+function toSampleRate(audio: unknown, fallback: number): number {
+  const sampleRate = (audio as { sampleRate?: unknown })?.sampleRate;
+  if (typeof sampleRate === "number" && Number.isFinite(sampleRate) && sampleRate > 0) {
+    return sampleRate;
+  }
+  return fallback;
+}
+
+function createSherpaOfflineTts(params: {
+  modelId: "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16";
+  modelDir: string;
+  speed: number;
+}): { tts: any; speed: number } {
+  const sherpa = loadSherpaOnnxNode();
+  const model =
+    params.modelId === "kokoro-en-v0_19"
+      ? {
+          kokoro: {
+            model: `${params.modelDir}/model.onnx`,
+            voices: `${params.modelDir}/voices.bin`,
+            tokens: `${params.modelDir}/tokens.txt`,
+            dataDir: `${params.modelDir}/espeak-ng-data`,
+            lengthScale: 1.0,
+          },
+        }
+      : {
+          kitten: {
+            model: `${params.modelDir}/model.fp16.onnx`,
+            voices: `${params.modelDir}/voices.bin`,
+            tokens: `${params.modelDir}/tokens.txt`,
+            dataDir: `${params.modelDir}/espeak-ng-data`,
+            lengthScale: 1.0,
+          },
+        };
+
+  return {
+    tts: new sherpa.OfflineTts({
+      model,
+      numThreads: 2,
+      provider: "cpu",
+      maxNumSentences: 1,
+    }),
+    speed: params.speed,
+  };
+}
+
+async function readStreamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream as AsyncIterable<Buffer | Uint8Array>) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks);
+}
+
+function toCsv(rows: GeneratedSample[]): string {
+  const header = [
+    "model_id",
+    "voice_label",
+    "speaker_id",
+    "sample_rate",
+    "duration_seconds",
+    "relative_path",
+  ];
+
+  const lines = [header.join(",")];
+  for (const row of rows) {
+    const cells = [
+      row.modelId,
+      row.voiceLabel,
+      row.speakerId === null ? "" : String(row.speakerId),
+      String(row.sampleRate),
+      row.durationSeconds.toFixed(3),
+      row.relativePath,
+    ];
+
+    lines.push(
+      cells
+        .map((cell) => {
+          const escaped = cell.replace(/"/g, "\"\"");
+          return escaped.includes(",") ? `"${escaped}"` : escaped;
+        })
+        .join(",")
+    );
+  }
+  return `${lines.join("\n")}\n`;
+}
+
+const logger = createRootLogger({ level: "info", format: "pretty" });
+const options = parseArgs(process.argv.slice(2));
+
+logger.info(
+  {
+    modelsDir: options.modelsDir,
+    outputDir: options.outputDir,
+    modelIds: options.modelIds,
+    speed: options.speed,
+    text: options.text,
+  },
+  "Generating TTS matrix across all model/voice combinations"
+);
+
+await ensureLocalSpeechModels({
+  modelsDir: options.modelsDir,
+  modelIds: options.modelIds,
+  autoDownload: true,
+  logger,
+});
+
+await mkdir(options.outputDir, { recursive: true });
+
+const generated: GeneratedSample[] = [];
+
+for (const modelId of options.modelIds) {
+  const modelDir = getLocalSpeechModelDir(options.modelsDir, modelId);
+  const modelOutputDir = path.join(options.outputDir, sanitizeFilePart(modelId));
+  await mkdir(modelOutputDir, { recursive: true });
+
+  if (modelId === "pocket-tts-onnx-int8") {
+    const tts = await PocketTtsOnnxTTS.create(
+      {
+        modelDir,
+        precision: "int8",
+        targetChunkMs: 50,
+      },
+      logger
+    );
+    const result = await tts.synthesizeSpeech(options.text);
+    const pcm16 = await readStreamToBuffer(result.stream);
+    const sampleRate = parsePcmRateFromFormat(result.format, 24000) ?? 24000;
+
+    const voiceLabel = "default";
+    const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`;
+    const absPath = path.join(modelOutputDir, filename);
+    const relPath = path.relative(options.outputDir, absPath);
+
+    await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate));
+
+    generated.push({
+      modelId,
+      voiceLabel,
+      speakerId: null,
+      sampleRate,
+      durationSeconds: pcm16.length / 2 / sampleRate,
+      relativePath: relPath,
+    });
+
+    logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample");
+    continue;
+  }
+
+  if (modelId !== "kokoro-en-v0_19" && modelId !== "kitten-nano-en-v0_1-fp16") {
+    throw new Error(`Unsupported local Sherpa TTS model for this script: ${modelId}`);
+  }
+
+  const { tts, speed } = createSherpaOfflineTts({
+    modelId,
+    modelDir,
+    speed: options.speed,
+  });
+
+  try {
+    const rawNumSpeakers = (tts as { numSpeakers?: unknown }).numSpeakers;
+    const numSpeakers =
+      typeof rawNumSpeakers === "number" && Number.isFinite(rawNumSpeakers) && rawNumSpeakers > 0
+        ? Math.floor(rawNumSpeakers)
+        : 1;
+    const pad = Math.max(2, String(Math.max(0, numSpeakers - 1)).length);
+    const fallbackSampleRate =
+      typeof (tts as { sampleRate?: unknown }).sampleRate === "number"
+        ? Math.max(1, Math.floor((tts as { sampleRate: number }).sampleRate))
+        : 24000;
+
+    logger.info({ modelId, numSpeakers }, "Discovered Sherpa speaker count");
+
+    for (let sid = 0; sid < numSpeakers; sid += 1) {
+      const audio = (tts as { generate: (config: { text: string; sid: number; speed: number }) => unknown }).generate({
+        text: options.text,
+        sid,
+        speed,
+      });
+      const samples = toFloatSamples(audio);
+      const sampleRate = toSampleRate(audio, fallbackSampleRate);
+      const pcm16 = float32ToPcm16le(samples);
+
+      const voiceLabel = `sid-${String(sid).padStart(pad, "0")}`;
+      const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`;
+      const absPath = path.join(modelOutputDir, filename);
+      const relPath = path.relative(options.outputDir, absPath);
+
+      await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate));
+
+      generated.push({
+        modelId,
+        voiceLabel,
+        speakerId: sid,
+        sampleRate,
+        durationSeconds: pcm16.length / 2 / sampleRate,
+        relativePath: relPath,
+      });
+
+      logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample");
+    }
+  } finally {
+    (tts as { free?: () => void }).free?.();
+  }
+}
+
+const manifest = {
+  createdAt: new Date().toISOString(),
+  text: options.text,
+  speed: options.speed,
+  modelsDir: options.modelsDir,
+  outputDir: options.outputDir,
+  namingConvention: "<model-id>/<model-id>__voice-<voice-label>.wav",
+  notes: [
+    "For Sherpa kitten/kokoro models, voice-label uses sid-<speaker-id>.",
+    "For Pocket TTS, voice-label is default.",
+  ],
+  generated,
+};
+
+await writeFile(
+  path.join(options.outputDir, "manifest.json"),
+  `${JSON.stringify(manifest, null, 2)}\n`
+);
+await writeFile(path.join(options.outputDir, "samples.csv"), toCsv(generated));
+await writeFile(
+  path.join(options.outputDir, "README.txt"),
+  [
+    "Sherpa TTS sample matrix",
+    "",
+    `Sample text: ${options.text}`,
+    `Speed: ${options.speed}`,
+    "",
+    "Naming convention:",
+    "  <model-id>/<model-id>__voice-<voice-label>.wav",
+    "  voice-label = sid-<speaker-id> for Sherpa models",
+    "  voice-label = default for Pocket TTS",
+    "",
+    `Total files: ${generated.length}`,
+  ].join("\n")
+);
+
+logger.info(
+  {
+    outputDir: options.outputDir,
+    generatedCount: generated.length,
+    models: Array.from(new Set(generated.map((item) => item.modelId))),
+  },
+  "TTS matrix generation complete"
+);

packages/server/src/server/agent/agent-manager.test.ts

@@ -150,7 +150,7 @@ describe("AgentManager", () => {
     ).rejects.toThrow("Working directory does not exist");
   });
 
-  test("resumeAgent keeps metadata config and applies systemPrompt/mcpServers overrides", async () => {
+  test("resumeAgentFromPersistence keeps metadata config and applies systemPrompt/mcpServers overrides", async () => {
     const workdir = mkdtempSync(join(tmpdir(), "agent-manager-resume-"));
     const storagePath = join(workdir, "agents");
     const storage = new AgentStorage(storagePath, logger);
@@ -211,7 +211,7 @@ describe("AgentManager", () => {
       },
     };
 
-    const resumed = await manager.resumeAgent(handle, {
+    const resumed = await manager.resumeAgentFromPersistence(handle, {
       cwd: workdir,
       systemPrompt: "new prompt",
       mcpServers: {
@@ -243,6 +243,92 @@ describe("AgentManager", () => {
     });
   });
 
+  test("reloadAgentSession preserves timeline and does not force history replay", async () => {
+    const workdir = mkdtempSync(join(tmpdir(), "agent-manager-reload-"));
+    const storagePath = join(workdir, "agents");
+    const storage = new AgentStorage(storagePath, logger);
+
+    class HistoryProbeSession extends TestAgentSession {
+      constructor(
+        config: AgentSessionConfig,
+        private readonly historyText: string | null
+      ) {
+        super(config);
+      }
+
+      async *streamHistory(): AsyncGenerator<AgentStreamEvent> {
+        if (!this.historyText) {
+          return;
+        }
+        yield {
+          type: "timeline",
+          provider: this.provider,
+          item: { type: "assistant_message", text: this.historyText },
+        };
+      }
+    }
+
+    class HistoryProbeClient implements AgentClient {
+      readonly provider = "codex" as const;
+      readonly capabilities = TEST_CAPABILITIES;
+
+      async isAvailable(): Promise<boolean> {
+        return true;
+      }
+
+      async createSession(config: AgentSessionConfig): Promise<AgentSession> {
+        return new HistoryProbeSession(config, null);
+      }
+
+      async resumeSession(
+        handle: AgentPersistenceHandle,
+        overrides?: Partial<AgentSessionConfig>
+      ): Promise<AgentSession> {
+        const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
+        const merged: AgentSessionConfig = {
+          ...metadata,
+          ...overrides,
+          provider: "codex",
+          cwd: overrides?.cwd ?? metadata.cwd ?? process.cwd(),
+        };
+        return new HistoryProbeSession(merged, "history replay from provider");
+      }
+    }
+
+    const manager = new AgentManager({
+      clients: {
+        codex: new HistoryProbeClient(),
+      },
+      registry: storage,
+      logger,
+      idFactory: () => "00000000-0000-4000-8000-000000000113",
+    });
+
+    const snapshot = await manager.createAgent({
+      provider: "codex",
+      cwd: workdir,
+    });
+
+    await manager.appendTimelineItem(snapshot.id, {
+      type: "assistant_message",
+      text: "keep this timeline in memory",
+    });
+    await manager.hydrateTimelineFromProvider(snapshot.id);
+    const beforeReload = manager.getTimeline(snapshot.id);
+    expect(beforeReload).toHaveLength(1);
+
+    await manager.reloadAgentSession(snapshot.id, {
+      systemPrompt: "reloaded prompt",
+    });
+    const afterReload = manager.getTimeline(snapshot.id);
+    expect(afterReload).toEqual(beforeReload);
+
+    // If reload resets historyPrimed, this would replay provider history and append another item.
+    await manager.hydrateTimelineFromProvider(snapshot.id);
+    const afterHydrate = manager.getTimeline(snapshot.id);
+    expect(afterHydrate).toEqual(beforeReload);
+  });
+
   test("createAgent fails when generated agent ID is not a UUID", async () => {
     const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
     const storagePath = join(workdir, "agents");

packages/server/src/server/agent/agent-manager.ts

@@ -371,7 +371,9 @@ export class AgentManager {
     );
   }
 
-  async resumeAgent(
+  // Reconstruct an agent from provider persistence. Callers should explicitly
+  // hydrate timeline history after resume.
+  async resumeAgentFromPersistence(
     handle: AgentPersistenceHandle,
     overrides?: Partial<AgentSessionConfig>,
     agentId?: string,
@@ -384,7 +386,7 @@ export class AgentManager {
   ): Promise<ManagedAgent> {
     const resolvedAgentId = validateAgentId(
       agentId ?? this.idFactory(),
-      "resumeAgent"
+      "resumeAgentFromPersistence"
     );
     const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
     const mergedConfig = {
@@ -407,11 +409,18 @@ export class AgentManager {
     );
   }
 
-  async refreshAgentFromPersistence(
+  // Hot-reload an active agent session with config overrides while preserving
+  // in-memory timeline state.
+  async reloadAgentSession(
     agentId: string,
     overrides?: Partial<AgentSessionConfig>
   ): Promise<ManagedAgent> {
     const existing = this.requireAgent(agentId);
+    const preservedTimeline = [...existing.timeline];
+    const preservedHistoryPrimed = existing.historyPrimed;
+    const preservedLastUsage = existing.lastUsage;
+    const preservedLastError = existing.lastError;
+    const preservedAttention = existing.attention;
     const handle = existing.persistence;
     const provider = handle?.provider ?? existing.provider;
     const client = this.requireClient(provider);
@@ -437,12 +446,17 @@ export class AgentManager {
       );
     }
 
-    // Preserve existing labels during refresh
+    // Preserve existing labels and timeline during reload.
     return this.registerSession(session, normalizedConfig, agentId, {
       labels: existing.labels,
       createdAt: existing.createdAt,
       updatedAt: existing.updatedAt,
       lastUserMessageAt: existing.lastUserMessageAt,
+      timeline: preservedTimeline,
+      historyPrimed: preservedHistoryPrimed,
+      lastUsage: preservedLastUsage,
+      lastError: preservedLastError,
+      attention: preservedAttention,
     });
   }
 
@@ -844,9 +858,9 @@ export class AgentManager {
     return iterator.done ? null : iterator.value;
   }
 
-  async primeAgentHistory(agentId: string): Promise<void> {
+  async hydrateTimelineFromProvider(agentId: string): Promise<void> {
     const agent = this.requireAgent(agentId);
-    await this.primeHistory(agent);
+    await this.hydrateTimeline(agent);
   }
 
   private getLastAssistantMessage(agentId: string): string | null {
@@ -1033,6 +1047,11 @@ export class AgentManager {
       updatedAt?: Date;
       lastUserMessageAt?: Date | null;
       labels?: Record<string, string>;
+      timeline?: AgentTimelineItem[];
+      historyPrimed?: boolean;
+      lastUsage?: AgentUsage;
+      lastError?: string;
+      attention?: AttentionState;
     }
   ): Promise<ManagedAgent> {
     const resolvedAgentId = validateAgentId(agentId, "registerSession");
@@ -1056,11 +1075,22 @@ export class AgentManager {
       currentModeId: null,
       pendingPermissions: new Map(),
       pendingRun: null,
-      timeline: [],
+      timeline: options?.timeline ? [...options.timeline] : [],
       persistence: attachPersistenceCwd(session.describePersistence(), config.cwd),
-      historyPrimed: false,
+      historyPrimed: options?.historyPrimed ?? false,
       lastUserMessageAt: options?.lastUserMessageAt ?? null,
-      attention: { requiresAttention: false },
+      lastUsage: options?.lastUsage,
+      lastError: options?.lastError,
+      attention:
+        options?.attention != null
+          ? options.attention.requiresAttention
+            ? {
+                requiresAttention: true,
+                attentionReason: options.attention.attentionReason,
+                attentionTimestamp: new Date(options.attention.attentionTimestamp),
+              }
+            : { requiresAttention: false }
+          : { requiresAttention: false },
       internal: config.internal ?? false,
       labels: options?.labels ?? {},
     } as ActiveManagedAgent;
@@ -1144,7 +1174,7 @@ export class AgentManager {
     }
   }
 
-  private async primeHistory(agent: ActiveManagedAgent): Promise<void> {
+  private async hydrateTimeline(agent: ActiveManagedAgent): Promise<void> {
     if (agent.historyPrimed) {
       return;
     }

packages/server/src/server/agent/agent-metadata-generator.test.ts

@@ -82,7 +82,7 @@ function initGitRepo(repoDir: string): void {
         modeId: "auto",
         cwd: repoDir,
         title: "Main Agent",
-      }, "metadata-title-agent");
+      }, "4e0a4508-e522-4fe9-8384-cf3bf889f16d");
 
       await generateAndApplyAgentMetadata({
         agentManager: manager,
@@ -122,7 +122,7 @@ function initGitRepo(repoDir: string): void {
         modeId: "auto",
         cwd: worktree.worktreePath,
         title: "Worktree Agent",
-      }, "metadata-branch-agent");
+      }, "32bb765d-f637-44a2-9820-f2efd5261418");
 
       await generateAndApplyAgentMetadata({
         agentManager: manager,

packages/server/src/server/agent/providers/claude-agent.test.ts

@@ -825,6 +825,7 @@ async function startAgentMcpServer(): Promise<AgentMcpServerHandle> {
       // Wait for history file to be written
       const historyPaths = getClaudeHistoryPaths(cwd, handle!.sessionId);
       expect(await waitForHistoryFile(historyPaths)).toBe(true);
+      expect(await waitForHistoryContains(historyPaths, secretWord)).toBe(true);
 
       // Resume and verify context is preserved
       const resumed = await client.resumeSession(handle!, { cwd });
@@ -1079,6 +1080,7 @@ async function startAgentMcpServer(): Promise<AgentMcpServerHandle> {
 
         const historyPaths = getClaudeHistoryPaths(cwd, sessionId!);
         expect(await waitForHistoryFile(historyPaths)).toBe(true);
+        expect(await waitForHistoryContains(historyPaths, "HYDRATION_PROOF_LINE_TWO")).toBe(true);
 
         const resumed = await client.resumeSession(handle!, { cwd });
         const hydrationUpdates: StreamHydrationUpdate[] = [];
@@ -1179,6 +1181,7 @@ async function startAgentMcpServer(): Promise<AgentMcpServerHandle> {
 
         const historyPaths = getClaudeHistoryPaths(cwd, sessionId!);
         expect(await waitForHistoryFile(historyPaths)).toBe(true);
+        expect(await waitForHistoryContains(historyPaths, promptMarker)).toBe(true);
 
         const resumed = await client.resumeSession(handle!, { cwd });
         const hydrationUpdates: StreamHydrationUpdate[] = [];
@@ -1524,6 +1527,43 @@ async function waitForHistoryFile(historyPaths: string | string[], timeoutMs = 1
   return candidates.some((entry) => existsSync(entry));
 }
 
+async function waitForHistoryContains(
+  historyPaths: string | string[],
+  marker: string,
+  timeoutMs = 15_000
+): Promise<boolean> {
+  const candidates = Array.isArray(historyPaths)
+    ? Array.from(new Set(historyPaths))
+    : [historyPaths];
+  const deadline = Date.now() + timeoutMs;
+
+  const hasMarker = (): boolean => {
+    for (const historyPath of candidates) {
+      if (!existsSync(historyPath)) {
+        continue;
+      }
+      try {
+        const content = readFileSync(historyPath, "utf8");
+        if (content.includes(marker)) {
+          return true;
+        }
+      } catch {
+        // History file may still be mid-write; retry.
+      }
+    }
+    return false;
+  };
+
+  while (Date.now() < deadline) {
+    if (hasMarker()) {
+      return true;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 250));
+  }
+
+  return hasMarker();
+}
+
 function rawContainsText(raw: unknown, text: string, depth = 0): boolean {
   if (!raw || typeof text !== "string" || !text) {
     return false;

packages/server/src/server/agent/providers/codex-app-server-agent.test.ts

@@ -1214,11 +1214,15 @@ describe("Codex app-server provider (integration)", () => {
             (item) => item.type === "assistant_message" || item.type === "user_message"
           )
         ).toBe(true);
-
+        const historyIncludesToken = history.some(
-        const response = await resumed.run(
+          (item) =>
-          `Respond with the exact token ${token} and stop.`
+            (item.type === "assistant_message" || item.type === "user_message") &&
+            item.text.includes(token)
         );
-        expect(response.finalText).toContain(token);
+        expect(historyIncludesToken).toBe(true);
+
+        const response = await resumed.run("Reply with CONTEXT_OK and stop.");
+        expect(response.finalText.toLowerCase()).toContain("context_ok");
 
         const resumedHandle = resumed.describePersistence();
         expect(resumedHandle?.sessionId).toBe(handle?.sessionId);
@@ -1327,34 +1331,51 @@ describe("Codex app-server provider (integration)", () => {
       );
 
       let failure: string | null = null;
-      for await (const event of events) {
+      try {
-        if (event.type === "permission_requested" && event.request.name === "CodexFileChange") {
+        await Promise.race([
-          sawPermission = true;
+          (async () => {
-          captured = event.request;
+            for await (const event of events) {
-          await session.respondToPermission(event.request.id, { behavior: "allow" });
+              if (event.type === "permission_requested" && event.request.name === "CodexFileChange") {
-        }
+                sawPermission = true;
-        if (
+                captured = event.request;
-          event.type === "permission_resolved" &&
+                await session.respondToPermission(event.request.id, { behavior: "allow" });
-          captured &&
+              }
-          event.requestId === captured.id &&
+              if (
-          event.resolution.behavior === "allow"
+                event.type === "permission_resolved" &&
-        ) {
+                captured &&
-          sawPermissionResolved = true;
+                event.requestId === captured.id &&
-        }
+                event.resolution.behavior === "allow"
-        if (event.type === "timeline" && event.item.type === "tool_call") {
+              ) {
-          timelineItems.push(event.item);
+                sawPermissionResolved = true;
-        }
+              }
-        if (event.type === "turn_failed") {
+              if (event.type === "timeline" && event.item.type === "tool_call") {
-          failure = event.error;
+                timelineItems.push(event.item);
-          break;
+              }
-        }
+              if (event.type === "turn_failed") {
-        if (event.type === "turn_completed") {
+                failure = event.error;
-          break;
+                break;
-        }
+              }
+              if (event.type === "turn_completed") {
+                break;
+              }
+            }
+          })(),
+          new Promise<never>((_, reject) =>
+            setTimeout(
+              () =>
+                reject(
+                  new Error(
+                    "Timed out waiting for Codex file approval flow to complete"
+                  )
+                ),
+              100_000
+            )
+          ),
+        ]);
+      } finally {
+        await session.close();
       }
 
-      await session.close();
-
       if (failure) {
         throw new Error(failure);
       }
@@ -1391,7 +1412,7 @@ describe("Codex app-server provider (integration)", () => {
       cleanup();
       rmSync(cwd, { recursive: true, force: true });
     }
-  }, 60000);
+  }, 120000);
 
   test.runIf(isCodexInstalled())("tool approval flow requests user input for app tools", async () => {
     const cleanup = useTempCodexSessionDir();

packages/server/src/server/daemon-e2e/git-operations.e2e.test.ts

@@ -424,7 +424,10 @@ describe("daemon E2E", () => {
         );
 
         expect(completed.callId).toBeTruthy();
-        expect(completed.output).toBeTruthy();
+        expect(completed.detail.type).toBe("unknown");
+        if (completed.detail.type === "unknown") {
+          expect(completed.detail.output).toBeTruthy();
+        }
         expect(existsSync(path.join(agent.cwd, "setup-done.txt"))).toBe(true);
 
         await ctx.client.deleteAgent(agent.id);
@@ -507,7 +510,7 @@ describe("daemon E2E", () => {
 
         expect(existsSync(path.join(agent.cwd, "setup-start.txt"))).toBe(true);
 
-        const output = failed.output as any;
+        const output = failed.detail.type === "unknown" ? failed.detail.output as any : undefined;
         const commands = output?.commands as any[] | undefined;
         expect(Array.isArray(commands)).toBe(true);
         expect(commands?.[0]?.exitCode).toBe(7);

packages/server/src/server/daemon-e2e/tool-calls.e2e.test.ts

@@ -18,6 +18,78 @@ function tmpCwd(): string {
 const CODEX_TEST_MODEL = "gpt-5.1-codex-mini";
 const CODEX_TEST_THINKING_OPTION_ID = "low";
 
+type ToolCallItem = Extract<AgentTimelineItem, { type: "tool_call" }>;
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return null;
+  }
+  return value as Record<string, unknown>;
+}
+
+function commandFromRawInput(value: unknown): string | null {
+  const record = asRecord(value);
+  if (!record) {
+    return null;
+  }
+
+  const commandValue = record.command ?? record.cmd;
+  if (typeof commandValue === "string" && commandValue.length > 0) {
+    return commandValue;
+  }
+  if (Array.isArray(commandValue)) {
+    const parts = commandValue.filter((entry): entry is string => typeof entry === "string");
+    if (parts.length > 0) {
+      return parts.join(" ");
+    }
+  }
+  return null;
+}
+
+function commandFromToolCallDetail(detail: ToolCallItem["detail"]): string | null {
+  if (detail.type === "shell") {
+    return detail.command;
+  }
+  if (detail.type === "unknown") {
+    return commandFromRawInput(detail.input);
+  }
+  return null;
+}
+
+function extractPathFromPayload(value: unknown): string | null {
+  const record = asRecord(value);
+  if (!record) {
+    return null;
+  }
+
+  const directPath = record.path ?? record.file_path ?? record.filePath;
+  if (typeof directPath === "string" && directPath.length > 0) {
+    return directPath;
+  }
+
+  if (Array.isArray(record.files)) {
+    for (const file of record.files) {
+      const fileRecord = asRecord(file);
+      const filePath = fileRecord?.path;
+      if (typeof filePath === "string" && filePath.length > 0) {
+        return filePath;
+      }
+    }
+  }
+
+  return null;
+}
+
+function pathFromToolCallDetail(detail: ToolCallItem["detail"]): string | null {
+  if (detail.type === "read" || detail.type === "edit" || detail.type === "write") {
+    return detail.filePath;
+  }
+  if (detail.type === "unknown") {
+    return extractPathFromPayload(detail.input) ?? extractPathFromPayload(detail.output);
+  }
+  return null;
+}
+
 describe("daemon E2E", () => {
   let ctx: DaemonTestContext;
 
@@ -34,9 +106,9 @@ describe("daemon E2E", () => {
     function extractToolCalls(
       queue: SessionOutboundMessage[],
       agentId: string
-    ): AgentTimelineItem[] {
+    ): ToolCallItem[] {
-      const byCallId = new Map<string, AgentTimelineItem>();
+      const byCallId = new Map<string, ToolCallItem>();
-      const noCallId: AgentTimelineItem[] = [];
+      const noCallId: ToolCallItem[] = [];
 
       for (const m of queue) {
         if (
@@ -58,9 +130,9 @@ describe("daemon E2E", () => {
     }
 
     // Helper to log tool call structure in a consistent format
-    function logToolCall(prefix: string, tc: AgentTimelineItem): void {
+    function logToolCall(prefix: string, tc: ToolCallItem): void {
-      if (tc.type !== "tool_call") return;
+      void prefix;
-
+      void tc;
     }
 
     test(
@@ -92,10 +164,15 @@ describe("daemon E2E", () => {
           logToolCall("CLAUDE_READ", tc);
         }
 
-        const readCall = toolCalls.find((tc) => tc.type === "tool_call" && tc.name === "Read");
+        const readCall = toolCalls.find((tc) => tc.name === "Read");
         expect(readCall).toBeDefined();
         expect(readCall?.name).toBe("Read");
-        expect(readCall?.input).toBeDefined();
+        expect(readCall?.detail).toBeDefined();
+        const readPath = readCall ? pathFromToolCallDetail(readCall.detail) : null;
+        expect(readPath).toBeTruthy();
+        if (readPath) {
+          expect(readPath).toContain("/etc/hosts");
+        }
 
         await ctx.client.deleteAgent(agent.id);
         collector.unsubscribe();
@@ -133,13 +210,15 @@ describe("daemon E2E", () => {
           logToolCall("CLAUDE_BASH", tc);
         }
 
-        const bashCall = toolCalls.find((tc) => tc.type === "tool_call" && tc.name === "Bash");
+        const bashCall = toolCalls.find((tc) => tc.name === "Bash");
         expect(bashCall).toBeDefined();
         expect(bashCall?.name).toBe("Bash");
-        expect(bashCall?.input).toBeDefined();
+        expect(bashCall?.detail).toBeDefined();
-        // Command text should be in input.command
+        const command = bashCall ? commandFromToolCallDetail(bashCall.detail) : null;
-        const bashInput = bashCall?.input as { command?: string } | undefined;
+        expect(command).toBeTruthy();
-        expect(bashInput?.command).toContain("echo");
+        if (command) {
+          expect(command).toContain("echo");
+        }
 
         await ctx.client.deleteAgent(agent.id);
         collector.unsubscribe();
@@ -179,9 +258,17 @@ describe("daemon E2E", () => {
           logToolCall("CLAUDE_EDIT", tc);
         }
 
-        const editCall = toolCalls.find((tc) => tc.type === "tool_call" && tc.name === "Edit");
+        const editCall = toolCalls.find((tc) => tc.name === "Edit");
         expect(editCall).toBeDefined();
-        expect(editCall?.input).toBeDefined();
+        expect(editCall?.detail).toBeDefined();
+        const editPath = editCall ? pathFromToolCallDetail(editCall.detail) : null;
+        if (editPath) {
+          expect(editPath).toBe(testFile);
+        } else if (editCall?.detail.type === "unknown") {
+          expect(editCall.detail.input ?? editCall.detail.output).toBeTruthy();
+        } else {
+          expect(editCall?.detail.type).toBe("edit");
+        }
 
         await ctx.client.deleteAgent(agent.id);
         collector.unsubscribe();
@@ -219,15 +306,12 @@ describe("daemon E2E", () => {
           logToolCall("CODEX_SHELL", tc);
         }
 
-        const shellCalls = toolCalls.filter(
+        const shellCalls = toolCalls.filter((tc) => tc.name === "shell");
-          (tc) => tc.type === "tool_call" && tc.name === "shell"
-        );
         expect(shellCalls.length).toBeGreaterThan(0);
 
         const echoCall = shellCalls.find((tc) => {
-          const shellInput = tc.input as { command?: string } | undefined;
+          const command = commandFromToolCallDetail(tc.detail);
-          return typeof shellInput?.command === "string" &&
+          return typeof command === "string" && command.includes("echo");
-            shellInput.command.includes("echo");
         });
         expect(echoCall).toBeDefined();
 

packages/server/src/server/session.ts

@@ -209,14 +209,7 @@ function deriveProjectGroupingName(projectKey: string): string {
   return segments[segments.length - 1] || projectKey;
 }
 
-function escapeXmlText(value: string): string {
+type ProcessingPhase = "idle" | "transcribing";
-  return value
-    .replace(/&/g, "&")
-    .replace(/</g, "&lt;")
-    .replace(/>/g, "&gt;");
-}
-
-type ProcessingPhase = "idle" | "transcribing" | "llm";
 
 type NormalizedGitOptions = {
   baseBranch?: string;
@@ -392,7 +385,6 @@ export class Session {
   // State machine
   private abortController: AbortController;
   private processingPhase: ProcessingPhase = "idle";
-  private currentStreamPromise: Promise<void> | null = null;
 
   // Voice mode state
   private isVoiceMode = false;
@@ -425,9 +417,6 @@ export class Session {
     { format: string; chunks: Buffer[] }
   >();
 
-  // Conversation history
-  private turnIndex = 0;
-
   // Per-session managers
   private readonly ttsManager: TTSManager;
   private readonly sttManager: STTManager;
@@ -519,7 +508,7 @@ export class Session {
     this.defaultLocalSpeechModelIds =
       dictation?.localModels?.defaultModelIds && dictation.localModels.defaultModelIds.length > 0
         ? [...new Set(dictation.localModels.defaultModelIds)]
-        : ["parakeet-tdt-0.6b-v2-int8", "pocket-tts-onnx-int8"];
+        : ["parakeet-tdt-0.6b-v2-int8", "kokoro-en-v0_19"];
     this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler;
     this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler;
     this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext;
@@ -641,6 +630,19 @@ export class Session {
     }
   }
 
+  private hasActiveAgentRun(agentId: string | null): boolean {
+    if (!agentId) {
+      return false;
+    }
+
+    const snapshot = this.agentManager.getAgent(agentId);
+    if (!snapshot) {
+      return false;
+    }
+
+    return snapshot.lifecycle === "running" || Boolean(snapshot.pendingRun);
+  }
+
   /**
    * Start streaming an agent run and forward results via the websocket broadcast
    */
@@ -739,6 +741,29 @@ export class Session {
           return;
         }
 
+        if (
+          this.isVoiceMode &&
+          this.voiceModeAgentId === event.agentId &&
+          event.event.type === "permission_requested" &&
+          isVoicePermissionAllowed(event.event.request)
+        ) {
+          const requestId = event.event.request.id;
+          void this.agentManager
+            .respondToPermission(event.agentId, requestId, {
+              behavior: "allow",
+            })
+            .catch((error) => {
+              this.sessionLogger.warn(
+                {
+                  err: error,
+                  agentId: event.agentId,
+                  requestId,
+                },
+                "Failed to auto-allow speak tool permission in voice mode"
+              );
+            });
+        }
+
         // Reduce bandwidth/CPU on mobile: only forward high-frequency agent stream events
         // for the focused agent, with a short grace window while backgrounded.
         //
@@ -874,7 +899,7 @@ export class Session {
       const handle = toAgentPersistenceHandle(this.sessionLogger, record.persistence);
       let snapshot: ManagedAgent;
       if (handle) {
-        snapshot = await this.agentManager.resumeAgent(
+        snapshot = await this.agentManager.resumeAgentFromPersistence(
           handle,
           buildConfigOverrides(record),
           agentId,
@@ -885,7 +910,7 @@ export class Session {
         snapshot = await this.agentManager.createAgent(config, agentId, { labels: record.labels });
       }
 
-      await this.agentManager.primeAgentHistory(agentId);
+      await this.agentManager.hydrateTimelineFromProvider(agentId);
       return this.agentManager.getAgent(agentId) ?? snapshot;
     })();
 
@@ -1592,7 +1617,7 @@ export class Session {
     };
 
     try {
-      const refreshed = await this.agentManager.refreshAgentFromPersistence(
+      const refreshed = await this.agentManager.reloadAgentSession(
         agentId,
         refreshOverrides
       );
@@ -1630,7 +1655,7 @@ export class Session {
     if (restoreAgentConfig && this.voiceModeBaseConfig) {
       const baseConfig = this.voiceModeBaseConfig;
       try {
-        await this.agentManager.refreshAgentFromPersistence(agentId, {
+        await this.agentManager.reloadAgentSession(agentId, {
           systemPrompt: buildVoiceModeSystemPrompt(baseConfig.systemPrompt, false),
           mcpServers: this.cloneMcpServers(baseConfig.mcpServers),
         });
@@ -2110,11 +2135,11 @@ export class Session {
       `Resuming agent ${handle.sessionId} (${handle.provider})`
     );
     try {
-      const snapshot = await this.agentManager.resumeAgent(
+      const snapshot = await this.agentManager.resumeAgentFromPersistence(
         handle,
         overrides
       );
-      await this.agentManager.primeAgentHistory(snapshot.id);
+      await this.agentManager.hydrateTimelineFromProvider(snapshot.id);
       await this.forwardAgentUpdate(snapshot);
       const timelineSize = this.emitAgentTimelineSnapshot(snapshot);
       if (requestId) {
@@ -2165,7 +2190,7 @@ export class Session {
       if (existing) {
         await this.interruptAgentIfRunning(agentId);
         if (existing.persistence) {
-          snapshot = await this.agentManager.refreshAgentFromPersistence(
+          snapshot = await this.agentManager.reloadAgentSession(
             agentId
           );
         } else {
@@ -2182,14 +2207,14 @@ export class Session {
             `Agent ${agentId} cannot be refreshed because it lacks persistence`
           );
         }
-        snapshot = await this.agentManager.resumeAgent(
+        snapshot = await this.agentManager.resumeAgentFromPersistence(
           handle,
           buildConfigOverrides(record),
           agentId,
           extractTimestamps(record)
         );
       }
-      await this.agentManager.primeAgentHistory(agentId);
+      await this.agentManager.hydrateTimelineFromProvider(agentId);
       await this.forwardAgentUpdate(snapshot);
       const timelineSize = this.emitAgentTimelineSnapshot(snapshot);
       if (requestId) {
@@ -5076,12 +5101,6 @@ export class Session {
     // Has content - abort any in-progress stream now
     this.createAbortController();
 
-    // Wait for aborted stream to finish cleanup (save partial response)
-    if (this.currentStreamPromise) {
-      this.sessionLogger.debug("Waiting for aborted stream to finish cleanup");
-      await this.currentStreamPromise;
-    }
-
     if (result.debugRecordingPath) {
       this.emit({
         type: "activity_log",
@@ -5114,19 +5133,27 @@ export class Session {
     });
 
     this.clearSpeechInProgress("transcription complete");
+    this.setPhase("idle");
     if (!this.isVoiceMode) {
       this.sessionLogger.debug(
         { requestId: result.requestId },
         "Skipping voice agent processing because voice mode is disabled"
       );
-      this.setPhase("idle");
       return;
     }
 
-    this.setPhase("llm");
+    const agentId = this.voiceModeAgentId;
-    this.currentStreamPromise = this.processVoiceTurn(result.text);
+    if (!agentId) {
-    await this.currentStreamPromise;
+      this.sessionLogger.warn(
-    this.setPhase("idle");
+        { requestId: result.requestId },
+        "Skipping voice agent processing because no agent is currently voice-enabled"
+      );
+      return;
+    }
+
+    // Route voice utterances through the same send path as regular text input:
+    // interrupt-if-running, record message, then start a new stream.
+    await this.handleSendAgentMessage(agentId, result.text);
   }
 
   private registerVoiceBridgeForAgent(agentId: string): void {
@@ -5168,109 +5195,6 @@ export class Session {
     });
   }
 
-  private buildVoiceAgentPrompt(userText: string): string {
-    const transcription = escapeXmlText(userText.trim());
-    return [
-      "<voice_transcription>",
-      transcription,
-      "</voice_transcription>",
-    ].join("\n");
-  }
-
-  private async processWithVoiceAgent(
-    agentId: string,
-    userText: string
-  ): Promise<void> {
-    await this.ensureAgentLoaded(agentId);
-    await this.interruptAgentIfRunning(agentId);
-
-    this.sessionLogger.info(
-      {
-        agentId,
-        userTextLength: userText.length,
-        userTextPreview: userText.slice(0, 160),
-      },
-      "Starting voice agent turn"
-    );
-
-    const prompt = this.buildVoiceAgentPrompt(userText);
-    this.agentManager.recordUserMessage(agentId, userText);
-
-    let sawSpeakToolCall = false;
-    const iterator = this.agentManager.streamAgent(agentId, prompt);
-    for await (const event of iterator) {
-      if (event.type === "turn_failed") {
-        throw new Error(event.error);
-      }
-      if (event.type === "timeline") {
-        if (event.item.type === "tool_call" && typeof event.item.name === "string") {
-          this.sessionLogger.info(
-            {
-              agentId,
-              toolName: event.item.name,
-              callId: event.item.callId,
-            },
-            "Voice agent timeline tool call"
-          );
-          if (event.item.name.toLowerCase().includes("speak")) {
-            sawSpeakToolCall = true;
-          }
-        }
-      }
-      if (event.type === "permission_requested") {
-        if (isVoicePermissionAllowed(event.request)) {
-          await this.agentManager.respondToPermission(agentId, event.request.id, {
-            behavior: "allow",
-          });
-        } else {
-          this.sessionLogger.info(
-            { agentId, requestName: event.request.name, requestId: event.request.id },
-            "Voice assistant left non-speak permission request for user decision"
-          );
-        }
-      }
-    }
-
-    if (!sawSpeakToolCall) {
-      this.sessionLogger.warn(
-        { agentId },
-        "Voice agent turn completed without speak tool call; no audio playback emitted"
-      );
-    } else {
-      this.sessionLogger.info(
-        { agentId },
-        "Voice agent turn included speak tool call"
-      );
-    }
-  }
-
-  /**
-   * Process user message through LLM with streaming and tool execution
-   */
-  private async processVoiceTurn(latestUserText?: string): Promise<void> {
-    try {
-      if (!this.isVoiceMode) {
-        this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled");
-        return;
-      }
-      const agentId = this.voiceModeAgentId;
-      if (!agentId) {
-        this.sessionLogger.warn(
-          "Ignoring processVoiceTurn call because no agent is currently voice-enabled"
-        );
-        return;
-      }
-      const normalized = (latestUserText ?? "").trim();
-      if (!normalized) {
-        return;
-      }
-      await this.processWithVoiceAgent(agentId, normalized);
-    } finally {
-      this.turnIndex++;
-      this.currentStreamPromise = null;
-    }
-  }
-
   /**
    * Handle abort request from client
    */
@@ -5280,29 +5204,32 @@ export class Session {
       `Abort request, phase: ${this.processingPhase}`
     );
 
-    if (this.processingPhase === "llm") {
+    this.abortController.abort();
-      // Already in LLM phase - abort and wait for cleanup
+    this.ttsManager.cancelPendingPlaybacks("abort request");
-      this.abortController.abort();
-      this.sessionLogger.debug("Aborted LLM processing");
 
-      // Wait for stream to finish saving partial response
+    // Voice abort should always interrupt active agent output immediately.
-      if (this.currentStreamPromise) {
+    if (this.isVoiceMode && this.voiceModeAgentId) {
-        this.sessionLogger.debug("Waiting for stream cleanup after abort");
+      try {
-        await this.currentStreamPromise;
+        await this.interruptAgentIfRunning(this.voiceModeAgentId);
+      } catch (error) {
+        this.sessionLogger.warn(
+          { err: error, agentId: this.voiceModeAgentId },
+          "Failed to interrupt active voice-mode agent on abort"
+        );
       }
+    }
 
-      // Reset phase to idle
+    if (this.processingPhase === "transcribing") {
-      this.setPhase("idle");
-
-      // Clear any pending segments and timeouts
-      this.pendingAudioSegments = [];
-      this.clearBufferTimeout();
-    } else if (this.processingPhase === "transcribing") {
       // Still in STT phase - we'll buffer the next audio
       this.sessionLogger.debug("Will buffer next audio (currently transcribing)");
       // Phase stays as 'transcribing', handleAudioChunk will handle buffering
+      return;
     }
-    // If idle, nothing to do
+
+    // Reset phase to idle and clear pending non-voice buffers.
+    this.setPhase("idle");
+    this.pendingAudioSegments = [];
+    this.clearBufferTimeout();
   }
 
   /**
@@ -5313,7 +5240,7 @@ export class Session {
   }
 
   /**
-   * Mark speech detection start and abort any active playback/LLM
+   * Mark speech detection start and abort any active playback/agent run.
    */
   private async handleVoiceSpeechStart(): Promise<void> {
     if (this.speechInProgress) {
@@ -5322,11 +5249,10 @@ export class Session {
 
     const chunkReceivedAt = Date.now();
     const phaseBeforeAbort = this.processingPhase;
-    const hadActiveStream = Boolean(this.currentStreamPromise);
+    const hadActiveStream = this.hasActiveAgentRun(this.voiceModeAgentId);
 
     this.speechInProgress = true;
-    this.sessionLogger.debug("Voice speech detected – aborting playback and LLM");
+    this.sessionLogger.debug("Voice speech detected – aborting playback and active agent run");
-    this.ttsManager.cancelPendingPlaybacks("voice speech detected");
 
     if (this.pendingAudioSegments.length > 0) {
       this.sessionLogger.debug(

packages/server/src/server/speech/providers/local/config.ts

@@ -144,6 +144,10 @@ export function resolveLocalSpeechConfig(params: {
       params.persisted.features?.voiceMode?.tts?.speed,
   });
 
+  const resolvedVoiceTtsSpeakerId =
+    parsed.voiceLocalTtsSpeakerId ??
+    (parsed.voiceLocalTtsModel === "kokoro-en-v0_19" ? 0 : undefined);
+
   return {
     local:
       parsed.includeProviderConfig
@@ -154,8 +158,8 @@ export function resolveLocalSpeechConfig(params: {
               dictationStt: parsed.dictationLocalSttModel,
               voiceStt: parsed.voiceLocalSttModel,
               voiceTts: parsed.voiceLocalTtsModel,
-              ...(parsed.voiceLocalTtsSpeakerId !== undefined
+              ...(resolvedVoiceTtsSpeakerId !== undefined
-                ? { voiceTtsSpeakerId: parsed.voiceLocalTtsSpeakerId }
+                ? { voiceTtsSpeakerId: resolvedVoiceTtsSpeakerId }
                 : {}),
               ...(parsed.voiceLocalTtsSpeed !== undefined
                 ? { voiceTtsSpeed: parsed.voiceLocalTtsSpeed }

packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts

@@ -74,6 +74,7 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
     requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
     description: "Kokoro TTS (higher quality; larger).",
     aliases: ["kokoro"],
+    defaultFor: "tts",
   },
   "pocket-tts-onnx-int8": {
     kind: "tts",
@@ -119,7 +120,6 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
     ],
     description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).",
     aliases: ["pocket", "pocket-tts"],
-    defaultFor: "tts",
   },
 } as const satisfies Record<string, SherpaOnnxCatalogEntry>;
 

packages/server/src/server/speech/speech-config-resolver.test.ts

@@ -36,12 +36,14 @@ describe("resolveSpeechConfig", () => {
       models: {
         dictationStt: "parakeet-tdt-0.6b-v2-int8",
         voiceStt: "parakeet-tdt-0.6b-v2-int8",
-        voiceTts: "pocket-tts-onnx-int8",
+        voiceTts: "kokoro-en-v0_19",
+        voiceTtsSpeakerId: 0,
       },
     });
     expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
     expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
-    expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
+    expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
+    expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0);
   });
 
   test("resolves feature-scoped local model env vars", () => {
@@ -122,6 +124,7 @@ describe("resolveSpeechConfig", () => {
 
     expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
     expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
-    expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
+    expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
+    expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0);
   });
 });

packages/website/src/routes/docs/voice.tsx

@@ -51,7 +51,7 @@ function VoiceDocs() {
         <h2 className="text-xl font-medium">Local Speech</h2>
         <p className="text-white/60 leading-relaxed">
           Local speech defaults to model IDs <code className="font-mono">parakeet-tdt-0.6b-v3-int8</code>{' '}
-          (STT) and <code className="font-mono">pocket-tts-onnx-int8</code> (TTS).
+          (STT) and <code className="font-mono">kokoro-en-v0_19</code> (TTS, speaker 0 / voice 00).
         </p>
         <p className="text-white/60 leading-relaxed">
           Missing models are downloaded at daemon startup into{' '}
@@ -66,7 +66,7 @@ function VoiceDocs() {
     "voiceMode": {
       "llm": { "provider": "claude", "model": "haiku" },
       "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8" },
-      "tts": { "provider": "local", "model": "pocket-tts-onnx-int8" }
+      "tts": { "provider": "local", "model": "kokoro-en-v0_19", "speakerId": 0 }
     }
   },
   "providers": {