Update files

packages/server/src/client/daemon-client.ts

@@ -201,6 +201,10 @@ type SpeechModelsListPayload = SpeechModelsListResponse["payload"];
 type SpeechModelsDownloadPayload = SpeechModelsDownloadResponse["payload"];
 type ListCommandsPayload = ListCommandsResponse["payload"];
 type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
+type SetVoiceModePayload = Extract<
+  SessionOutboundMessage,
+  { type: "set_voice_mode_response" }
+>["payload"];
 type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
 type ListTerminalsPayload = ListTerminalsResponse["payload"];
 type CreateTerminalPayload = CreateTerminalResponse["payload"];
@@ -1285,8 +1289,31 @@ export class DaemonClient {
   // Audio / Voice
   // ============================================================================
 
-  async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise<void> {
-    this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId });
+  async setVoiceMode(
+    enabled: boolean,
+    voiceAgentId?: string
+  ): Promise<SetVoiceModePayload> {
+    const requestId = this.createRequestId();
+    const message = SessionInboundMessageSchema.parse({
+      type: "set_voice_mode",
+      enabled,
+      ...(voiceAgentId ? { voiceAgentId } : {}),
+      requestId,
+    });
+    return this.sendRequest({
+      requestId,
+      message,
+      timeout: 10000,
+      select: (msg) => {
+        if (msg.type !== "set_voice_mode_response") {
+          return null;
+        }
+        if (msg.payload.requestId !== requestId) {
+          return null;
+        }
+        return msg.payload;
+      },
+    });
   }
 
   async sendVoiceAudioChunk(

packages/server/src/server/agent/tts-manager.test.ts

@@ -43,5 +43,83 @@ describe("TTSManager", () => {
     expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1);
     expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true);
   });
-});
 
+  it("splits long text into safe synthesis segments", async () => {
+    const calls: string[] = [];
+    const tts: TextToSpeechProvider = {
+      async synthesizeSpeech(text: string): Promise<{ stream: Readable; format: string }> {
+        calls.push(text);
+        return {
+          stream: Readable.from([Buffer.from("x")]),
+          format: "pcm;rate=24000",
+        };
+      },
+    };
+
+    const manager = new TTSManager("s1", pino({ level: "silent" }), tts);
+    const abort = new AbortController();
+    const longText = Array.from({ length: 180 })
+      .map((_, i) => `Sentence ${i + 1}.`)
+      .join(" ");
+
+    await manager.generateAndWaitForPlayback(
+      longText,
+      (msg) => {
+        if (msg.type === "audio_output") {
+          manager.confirmAudioPlayed(msg.payload.id);
+        }
+      },
+      abort.signal,
+      true
+    );
+
+    expect(calls.length).toBeGreaterThan(1);
+    expect(calls.every((text) => text.length <= 400)).toBe(true);
+  });
+
+  it("does not emit unhandled rejections when stream iteration fails", async () => {
+    const unhandled: unknown[] = [];
+    const onUnhandled = (reason: unknown) => {
+      unhandled.push(reason);
+    };
+    process.on("unhandledRejection", onUnhandled);
+
+    try {
+      const tts: TextToSpeechProvider = {
+        async synthesizeSpeech(): Promise<{ stream: Readable; format: string }> {
+          const stream = Readable.from(
+            (async function* () {
+              yield Buffer.from("a");
+              throw new Error("stream exploded");
+            })()
+          );
+          return {
+            stream,
+            format: "pcm;rate=24000",
+          };
+        },
+      };
+
+      const manager = new TTSManager("s1", pino({ level: "silent" }), tts);
+      const abort = new AbortController();
+
+      await expect(
+        manager.generateAndWaitForPlayback(
+          "hello",
+          (msg) => {
+            if (msg.type === "audio_output") {
+              manager.confirmAudioPlayed(msg.payload.id);
+            }
+          },
+          abort.signal,
+          true
+        )
+      ).rejects.toThrow("stream exploded");
+
+      await new Promise((resolve) => setTimeout(resolve, 0));
+      expect(unhandled).toHaveLength(0);
+    } finally {
+      process.off("unhandledRejection", onUnhandled);
+    }
+  });
+});

packages/server/src/server/agent/tts-manager.ts

@@ -10,6 +10,84 @@ interface PendingPlayback {
   streamEnded: boolean;
 }
 
+const MAX_TTS_SEGMENT_CHARS = 400;
+
+function splitTextForTts(text: string, maxChars: number): string[] {
+  const normalized = text.trim().replace(/\s+/g, " ");
+  if (!normalized) {
+    throw new Error("Cannot synthesize empty text");
+  }
+
+  if (normalized.length <= maxChars) {
+    return [normalized];
+  }
+
+  const parts: string[] = [];
+  const sentenceChunks = normalized.split(/(?<=[.!?])\s+/);
+
+  let current = "";
+  const pushCurrent = () => {
+    const trimmed = current.trim();
+    if (trimmed.length > 0) {
+      parts.push(trimmed);
+    }
+    current = "";
+  };
+
+  const appendFragment = (fragment: string) => {
+    const trimmed = fragment.trim();
+    if (!trimmed) {
+      return;
+    }
+
+    if (!current) {
+      current = trimmed;
+      return;
+    }
+
+    const candidate = `${current} ${trimmed}`;
+    if (candidate.length <= maxChars) {
+      current = candidate;
+      return;
+    }
+
+    pushCurrent();
+    current = trimmed;
+  };
+
+  const splitLargeFragment = (fragment: string): string[] => {
+    const trimmed = fragment.trim();
+    if (trimmed.length <= maxChars) {
+      return [trimmed];
+    }
+
+    const out: string[] = [];
+    let remaining = trimmed;
+    while (remaining.length > maxChars) {
+      let idx = remaining.lastIndexOf(" ", maxChars);
+      if (idx < Math.floor(maxChars * 0.5)) {
+        idx = maxChars;
+      }
+      out.push(remaining.slice(0, idx).trim());
+      remaining = remaining.slice(idx).trim();
+    }
+    if (remaining.length > 0) {
+      out.push(remaining);
+    }
+    return out;
+  };
+
+  for (const sentence of sentenceChunks) {
+    const fragments = splitLargeFragment(sentence);
+    for (const fragment of fragments) {
+      appendFragment(fragment);
+    }
+  }
+  pushCurrent();
+
+  return parts;
+}
+
 /**
  * Per-session TTS manager
  * Handles TTS audio generation and playback confirmation tracking
@@ -33,6 +111,37 @@ export class TTSManager {
     emitMessage: (msg: SessionOutboundMessage) => void,
     abortSignal: AbortSignal,
     isVoiceMode: boolean
+  ): Promise<void> {
+    this.logger.info(
+      {
+        isVoiceMode,
+        textLength: text.length,
+        text,
+      },
+      "TTS input text"
+    );
+
+    const segments = splitTextForTts(text, MAX_TTS_SEGMENT_CHARS);
+    for (const segment of segments) {
+      if (abortSignal.aborted) {
+        this.logger.debug("Aborted before generating segmented audio");
+        return;
+      }
+
+      await this.generateSegmentAndWaitForPlayback(
+        segment,
+        emitMessage,
+        abortSignal,
+        isVoiceMode
+      );
+    }
+  }
+
+  private async generateSegmentAndWaitForPlayback(
+    text: string,
+    emitMessage: (msg: SessionOutboundMessage) => void,
+    abortSignal: AbortSignal,
+    isVoiceMode: boolean
   ): Promise<void> {
     if (!this.tts) {
       throw new Error("TTS not configured");
@@ -121,11 +230,6 @@ export class TTSManager {
             },
           });
 
-          this.logger.debug(
-            { chunkId, isLastChunk: next.done },
-            "Sent audio chunk"
-          );
-
           chunkIndex += 1;
 
           if (next.done) {
@@ -151,7 +255,6 @@ export class TTSManager {
       } else {
         this.logger.error({ err: error }, "Error streaming audio");
         this.pendingPlaybacks.delete(audioId);
-        pendingPlayback.reject(error as Error);
         throw error;
       }
     } finally {

packages/server/src/server/daemon-client.e2e.test.ts

@@ -147,8 +147,16 @@ describe("daemon client E2E", () => {
     });
     expect(Array.isArray(voiceAgents)).toBe(true);
 
-    await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined();
-    await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined();
+    await expect(ctx.client.setVoiceMode(true)).resolves.toMatchObject({
+      enabled: true,
+      accepted: true,
+      error: null,
+    });
+    await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({
+      enabled: false,
+      accepted: true,
+      error: null,
+    });
 
     await ctx.client.deleteAgent(randomUUID());
   }, 30000);

packages/server/src/server/session.ts

@@ -879,7 +879,7 @@ export class Session {
           break;
 
         case "set_voice_mode":
-          await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId);
+          await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId, msg.requestId);
           break;
 
         case "send_agent_message_request":
@@ -1259,7 +1259,8 @@ export class Session {
    */
   private async handleSetVoiceMode(
     enabled: boolean,
-    voiceAgentId?: string
+    voiceAgentId?: string,
+    requestId?: string
   ): Promise<void> {
     if (enabled) {
       let normalizedVoiceAgentId: string | null = null;
@@ -1282,6 +1283,18 @@ export class Session {
         },
         "Voice mode enabled (agent-backed)"
       );
+      if (requestId) {
+        this.emit({
+          type: "set_voice_mode_response",
+          payload: {
+            requestId,
+            enabled: true,
+            voiceAgentId: this.voiceAssistantAgentId,
+            accepted: true,
+            error: null,
+          },
+        });
+      }
       return;
     }
 
@@ -1290,6 +1303,18 @@ export class Session {
       { voiceAssistantAgentId: this.voiceAssistantAgentId },
       "Voice mode disabled (agent-backed)"
     );
+    if (requestId) {
+      this.emit({
+        type: "set_voice_mode_response",
+        payload: {
+          requestId,
+          enabled: false,
+          voiceAgentId: this.voiceAssistantAgentId,
+          accepted: true,
+          error: null,
+        },
+      });
+    }
   }
 
   private parseVoiceAgentId(rawId: string, source: string): string {
@@ -2471,7 +2496,6 @@ export class Session {
     lastActivityAt: string;
     appVisible: boolean;
   }): void {
-    this.sessionLogger.debug({ heartbeat: msg }, "Client heartbeat");
     this.clientActivity = {
       deviceType: msg.deviceType,
       focusedAgentId: msg.focusedAgentId,
@@ -4152,6 +4176,12 @@ export class Session {
   private async handleAudioChunk(
     msg: Extract<SessionInboundMessage, { type: "voice_audio_chunk" }>
   ): Promise<void> {
+    if (!this.isVoiceMode) {
+      this.sessionLogger.warn(
+        "Received voice_audio_chunk while voice mode is disabled; transcript will be emitted but voice assistant turn is skipped"
+      );
+    }
+
     await this.handleVoiceSpeechStart();
 
     const chunkBuffer = Buffer.from(msg.audio, "base64");
@@ -4193,11 +4223,6 @@ export class Session {
       this.audioBuffer.totalPCMBytes += chunkBuffer.length;
     }
 
-    this.sessionLogger.debug(
-      { bytes: chunkBuffer.length, chunks: this.audioBuffer.chunks.length, pcmBytes: this.audioBuffer.totalPCMBytes },
-      `Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})`
-    );
-
     // In voice mode, only process audio when the user has finished speaking (isLast = true)
     // This prevents partial transcriptions from being sent to the LLM
     if (this.isVoiceMode) {
@@ -4343,6 +4368,15 @@ export class Session {
       });
 
       const transcriptText = result.text.trim();
+      this.sessionLogger.info(
+        {
+          requestId,
+          isVoiceMode: this.isVoiceMode,
+          transcriptLength: transcriptText.length,
+          transcript: transcriptText,
+        },
+        "Transcription result"
+      );
 
       // Emit transcription result
       this.emit({
@@ -4408,8 +4442,17 @@ export class Session {
         },
       });
 
-      // Set phase to LLM and process (TTS enabled in voice mode for voice agents)
       this.clearSpeechInProgress("transcription complete");
+      if (!this.isVoiceMode) {
+        this.sessionLogger.debug(
+          { requestId },
+          "Skipping voice agent processing because voice mode is disabled"
+        );
+        this.setPhase("idle");
+        return;
+      }
+
+      // Set phase to LLM and process (TTS enabled in voice mode for voice agents)
       this.setPhase("llm");
       this.currentStreamPromise = this.processVoiceTurn(result.text);
       await this.currentStreamPromise;
@@ -4560,7 +4603,6 @@ export class Session {
     this.agentManager.recordUserMessage(agentId, userText);
 
     let sawSpeakToolCall = false;
-    const assistantTextChunks: string[] = [];
     const iterator = this.agentManager.streamAgent(agentId, prompt);
     for await (const event of iterator) {
       if (event.type === "turn_failed") {
@@ -4572,9 +4614,6 @@ export class Session {
             sawSpeakToolCall = true;
           }
         }
-        if (event.item.type === "assistant_message" && event.item.text.trim().length > 0) {
-          assistantTextChunks.push(event.item.text.trim());
-        }
       }
       if (event.type === "permission_requested") {
         if (this.shouldAllowVoicePermission(event.request)) {
@@ -4594,26 +4633,10 @@ export class Session {
       }
     }
 
-    if (!sawSpeakToolCall && assistantTextChunks.length > 0) {
-      const fallbackText = assistantTextChunks.join(" ").trim();
-      await this.ttsManager.generateAndWaitForPlayback(
-        fallbackText,
-        (msg) => this.emit(msg),
-        this.abortController.signal,
-        true
-      );
-      this.emit({
-        type: "activity_log",
-        payload: {
-          id: uuidv4(),
-          timestamp: new Date(),
-          type: "assistant",
-          content: fallbackText,
-        },
-      });
+    if (!sawSpeakToolCall) {
       this.sessionLogger.warn(
         { voiceAssistantAgentId: agentId },
-        "Voice agent responded without speak tool; used fallback TTS from assistant text"
+        "Voice agent turn completed without speak tool call; no audio playback emitted"
       );
     }
   }

packages/server/src/server/websocket-server.ts

@@ -389,7 +389,14 @@ export class VoiceAssistantWebSocketServer {
           ? { sessionMessageType: message.message.type }
           : {}),
       };
-      this.logger.debug(messageSummary, "Received message");
+      const isSessionNoise =
+        message.type === "session" &&
+        (message.message.type === "client_heartbeat" ||
+          message.message.type === "voice_audio_chunk" ||
+          message.message.type === "dictation_stream_chunk");
+      if (!isSessionNoise) {
+        this.logger.debug(messageSummary, "Received message");
+      }
 
       if (message.type === "ping") {
         this.sendToClient(ws, { type: "pong" });

packages/server/src/shared/messages.ts

@@ -348,6 +348,7 @@ export const SetVoiceModeMessageSchema = z.object({
   type: z.literal("set_voice_mode"),
   enabled: z.boolean(),
   voiceAgentId: z.string().optional(),
+  requestId: z.string().optional(),
 });
 
 export const SendAgentMessageSchema = z.object({
@@ -571,6 +572,17 @@ export const SetAgentThinkingResponseMessageSchema = z.object({
   }),
 });
 
+export const SetVoiceModeResponseMessageSchema = z.object({
+  type: z.literal("set_voice_mode_response"),
+  payload: z.object({
+    requestId: z.string(),
+    enabled: z.boolean(),
+    voiceAgentId: z.string().nullable(),
+    accepted: z.boolean(),
+    error: z.string().nullable(),
+  }),
+});
+
 export const AgentPermissionResponseMessageSchema = z.object({
   type: z.literal("agent_permission_response"),
   agentId: z.string(),
@@ -1627,6 +1639,7 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
   FetchAgentsResponseMessageSchema,
   FetchAgentResponseMessageSchema,
   SendAgentMessageResponseMessageSchema,
+  SetVoiceModeResponseMessageSchema,
   SetAgentModeResponseMessageSchema,
   SetAgentModelResponseMessageSchema,
   SetAgentThinkingResponseMessageSchema,
@@ -1690,6 +1703,9 @@ export type FetchAgentResponseMessage = z.infer<
 export type SendAgentMessageResponseMessage = z.infer<
   typeof SendAgentMessageResponseMessageSchema
 >;
+export type SetVoiceModeResponseMessage = z.infer<
+  typeof SetVoiceModeResponseMessageSchema
+>;
 export type WaitForFinishResponseMessage = z.infer<
   typeof WaitForFinishResponseMessageSchema
 >;