```

feat: add realtime mode drift protection and improve voice UX
```

packages/server/src/server/agent/tts-manager.ts

@@ -26,7 +26,8 @@ export class TTSManager {
   public async generateAndWaitForPlayback(
     text: string,
     emitMessage: (msg: SessionOutboundMessage) => void,
-    abortSignal: AbortSignal
+    abortSignal: AbortSignal,
+    isRealtimeMode: boolean
   ): Promise<void> {
     if (abortSignal.aborted) {
       console.log(
@@ -78,13 +79,14 @@ export class TTSManager {
       });
     }
 
-    // Emit audio output message
+    // Emit audio output message (include mode for drift protection)
     emitMessage({
       type: "audio_output",
       payload: {
         id: audioId,
         audio: audio.toString("base64"),
         format,
+        isRealtimeMode,
       },
     });
 

packages/server/src/server/messages.ts

@@ -7,6 +7,7 @@ import { z } from "zod";
 export const UserTextMessageSchema = z.object({
   type: z.literal("user_text"),
   text: z.string(),
+  disableTTS: z.boolean().optional(),
 });
 
 export const AudioChunkMessageSchema = z.object({
@@ -39,6 +40,11 @@ export const DeleteConversationRequestMessageSchema = z.object({
   conversationId: z.string(),
 });
 
+export const SetRealtimeModeMessageSchema = z.object({
+  type: z.literal("set_realtime_mode"),
+  enabled: z.boolean(),
+});
+
 export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
   UserTextMessageSchema,
   AudioChunkMessageSchema,
@@ -47,6 +53,7 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
   LoadConversationRequestMessageSchema,
   ListConversationsRequestMessageSchema,
   DeleteConversationRequestMessageSchema,
+  SetRealtimeModeMessageSchema,
 ]);
 
 export type SessionInboundMessage = z.infer<typeof SessionInboundMessageSchema>;
@@ -88,6 +95,7 @@ export const AudioOutputMessageSchema = z.object({
     audio: z.string(), // base64 encoded
     format: z.string(),
     id: z.string(),
+    isRealtimeMode: z.boolean(), // Mode when audio was generated (for drift protection)
   }),
 });
 
@@ -164,6 +172,12 @@ export const AgentStatusMessageSchema = z.object({
       type: z.literal("claude"),
       sessionId: z.string().optional(),
       error: z.string().optional(),
+      currentModeId: z.string().optional(),
+      availableModes: z.array(z.object({
+        id: z.string(),
+        name: z.string(),
+        description: z.string().nullable().optional(),
+      })).optional(),
     }),
   }),
 });

packages/server/src/server/session.ts

@@ -54,6 +54,9 @@ export class Session {
   private processingPhase: ProcessingPhase = "idle";
   private currentStreamPromise: Promise<void> | null = null;
 
+  // Realtime mode state
+  private isRealtimeMode = false;
+
   // Audio buffering for interruption handling
   private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [];
   private bufferTimeout: NodeJS.Timeout | null = null;
@@ -165,12 +168,12 @@ export class Session {
                   info: {
                     id: info.id,
                     status: info.status,
-                    createdAt: info.createdAt.toISOString(),
+                    createdAt: info.createdAt,
                     type: info.type,
-                    sessionId: info.sessionId,
-                    error: info.error,
-                    currentModeId: info.currentModeId,
-                    availableModes: info.availableModes,
+                    sessionId: info.sessionId ?? undefined,
+                    error: info.error ?? undefined,
+                    currentModeId: info.currentModeId ?? undefined,
+                    availableModes: info.availableModes ?? undefined,
                   },
                 },
               });
@@ -300,6 +303,10 @@ export class Session {
         case "delete_conversation_request":
           await this.handleDeleteConversation(msg.conversationId);
           break;
+
+        case "set_realtime_mode":
+          this.handleSetRealtimeMode(msg.enabled);
+          break;
       }
     } catch (error: any) {
       console.error(
@@ -400,6 +407,16 @@ export class Session {
     }
   }
 
+  /**
+   * Handle realtime mode toggle
+   */
+  private handleSetRealtimeMode(enabled: boolean): void {
+    this.isRealtimeMode = enabled;
+    console.log(
+      `[Session ${this.clientId}] Realtime mode ${enabled ? "enabled" : "disabled"}`
+    );
+  }
+
   /**
    * Send current session state (live agents and commands) to client
    */
@@ -471,8 +488,8 @@ export class Session {
     // Add to conversation
     this.messages.push({ role: "user", content: text });
 
-    // Process through LLM (TTS enabled for text input)
-    this.currentStreamPromise = this.processWithLLM(true);
+    // Process through LLM (TTS enabled in realtime mode for voice conversations)
+    this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode);
     await this.currentStreamPromise;
   }
 
@@ -618,9 +635,9 @@ export class Session {
       // Add to conversation
       this.messages.push({ role: "user", content: result.text });
 
-      // Set phase to LLM and process
+      // Set phase to LLM and process (TTS enabled in realtime mode for voice conversations)
       this.setPhase("llm");
-      this.currentStreamPromise = this.processWithLLM(true); // Enable TTS for voice input
+      this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode);
       await this.currentStreamPromise;
       this.setPhase("idle");
     } catch (error: any) {
@@ -648,12 +665,14 @@ export class Session {
 
     const flushTextBuffer = () => {
       if (textBuffer.length > 0) {
-        // TTS handling
+        // TTS handling (capture mode at generation time for drift protection)
         if (enableTTS) {
+          const modeAtGeneration = this.isRealtimeMode;
           pendingTTS = this.ttsManager.generateAndWaitForPlayback(
             textBuffer,
             (msg) => this.emit(msg),
-            this.abortController.signal
+            this.abortController.signal,
+            modeAtGeneration
           );
         }