```

feat: add realtime mode drift protection and improve voice UX
```

packages/app/src/app/index.tsx

@@ -53,6 +53,7 @@ import {
   ArrowUp,
   Square,
   AudioLines,
+  MicOff,
 } from "lucide-react-native";
 import type {
   ActivityLogPayload,
@@ -231,8 +232,14 @@ export default function VoiceAssistantScreen() {
 
   // Realtime mode state (defined early so we can use it in audioRecorder)
   const [isRealtimeMode, setIsRealtimeMode] = useState(false);
+  const isRealtimeModeRef = useRef(isRealtimeMode);
   const pulseAnim = useRef(new Animated.Value(1)).current;
 
+  // Keep ref in sync with state
+  useEffect(() => {
+    isRealtimeModeRef.current = isRealtimeMode;
+  }, [isRealtimeMode]);
+
   const insets = useSafeAreaInsets();
   const audioRecorder = useAudioRecorder();
   const audioPlayer = useAudioPlayer();
@@ -293,7 +300,7 @@ export default function VoiceAssistantScreen() {
       ]);
     },
     volumeThreshold: 0.3,
-    silenceDuration: 1000,
+    silenceDuration: 2000,
     speechConfirmationDuration: 300,
     detectionGracePeriod: 200,
   });
@@ -452,6 +459,11 @@ export default function VoiceAssistantScreen() {
       if (message.type !== "activity_log") return;
       const data = message.payload;
 
+      // Filter out transcription activity logs
+      if (data.type === "system" && data.content.includes("Transcribing")) {
+        return;
+      }
+
       // Handle tool calls
       if (data.type === "tool_call" && data.metadata) {
         const {
@@ -593,6 +605,42 @@ export default function VoiceAssistantScreen() {
       if (message.type !== "audio_output") return;
       const data = message.payload;
 
+      const currentIsRealtimeMode = isRealtimeModeRef.current;
+
+      // Drift protection: Don't play audio generated in different mode
+      if (data.isRealtimeMode !== currentIsRealtimeMode) {
+        console.log(
+          `[App] Skipping audio playback due to mode drift (generated in ${data.isRealtimeMode ? "realtime" : "normal"} mode, currently in ${currentIsRealtimeMode ? "realtime" : "normal"} mode)`
+        );
+
+        // Still send confirmation to prevent server from waiting
+        const confirmMessage: WSInboundMessage = {
+          type: "session",
+          message: {
+            type: "audio_played",
+            id: data.id,
+          },
+        };
+        ws.send(confirmMessage);
+        return;
+      }
+
+      // Additional check: Don't play if NOT in realtime mode (shouldn't happen with server-side fix, but defense in depth)
+      if (!currentIsRealtimeMode) {
+        console.log("[App] Skipping audio playback - not in realtime mode");
+
+        // Still send confirmation
+        const confirmMessage: WSInboundMessage = {
+          type: "session",
+          message: {
+            type: "audio_played",
+            id: data.id,
+          },
+        };
+        ws.send(confirmMessage);
+        return;
+      }
+
       try {
         setIsPlayingAudio(true);
 
@@ -669,18 +717,7 @@ export default function VoiceAssistantScreen() {
     // Conversation loaded handler
     const unsubConversationLoaded = ws.on("conversation_loaded", (message) => {
       if (message.type !== "conversation_loaded") return;
-      const { conversationId, messageCount } = message.payload;
+      // Don't show message in UI
-
-      setMessages((prev) => [
-        ...prev,
-        {
-          type: "activity",
-          id: generateMessageId(),
-          timestamp: Date.now(),
-          activityType: "success",
-          message: `Loaded conversation with ${messageCount} messages`,
-        },
-      ]);
     });
 
     // Artifact handler
@@ -734,32 +771,6 @@ export default function VoiceAssistantScreen() {
     };
   }, [ws, audioPlayer]);
 
-  // Connection status handler
-  useEffect(() => {
-    if (ws.isConnected) {
-      setMessages((prev) => [
-        ...prev,
-        {
-          type: "activity",
-          id: generateMessageId(),
-          timestamp: Date.now(),
-          activityType: "success",
-          message: "WebSocket connected",
-        },
-      ]);
-    } else if (messages.length > 0) {
-      setMessages((prev) => [
-        ...prev,
-        {
-          type: "activity",
-          id: generateMessageId(),
-          timestamp: Date.now(),
-          activityType: "error",
-          message: "WebSocket disconnected",
-        },
-      ]);
-    }
-  }, [ws.isConnected]);
 
   // Voice button handler
   async function handleVoicePress() {
@@ -778,16 +789,6 @@ export default function VoiceAssistantScreen() {
         );
 
         setIsProcessingAudio(true);
-        setMessages((prev) => [
-          ...prev,
-          {
-            type: "activity",
-            id: generateMessageId(),
-            timestamp: Date.now(),
-            activityType: "info",
-            message: "Sending audio to server...",
-          },
-        ]);
 
         // Convert to base64
         const arrayBuffer = await audioBlob.arrayBuffer();
@@ -974,6 +975,16 @@ export default function VoiceAssistantScreen() {
         setIsRealtimeMode(true);
         console.log("[App] Realtime mode enabled");
 
+        // Notify server of mode change
+        const modeMessage: WSInboundMessage = {
+          type: "session",
+          message: {
+            type: "set_realtime_mode",
+            enabled: true,
+          },
+        };
+        ws.send(modeMessage);
+
         setMessages((prev) => [
           ...prev,
           {
@@ -1004,6 +1015,16 @@ export default function VoiceAssistantScreen() {
         setIsRealtimeMode(false);
         console.log("[App] Realtime mode disabled");
 
+        // Notify server of mode change
+        const modeMessage: WSInboundMessage = {
+          type: "session",
+          message: {
+            type: "set_realtime_mode",
+            enabled: false,
+          },
+        };
+        ws.send(modeMessage);
+
         setMessages((prev) => [
           ...prev,
           {
@@ -1204,13 +1225,49 @@ export default function VoiceAssistantScreen() {
           ]}
         >
           {isRealtimeMode ? (
-            // Realtime mode - show volume meter
+            // Realtime mode - show volume meter and mute button
-            <Pressable
+            <View style={[styles.inputArea, { minHeight: 200 }]}>
-              onPress={handleRealtimeToggle}
+              <View style={{ flex: 1, justifyContent: "center", alignItems: "center" }}>
-              style={[styles.inputArea, { minHeight: 200, justifyContent: "center" }]}
+                <VolumeMeter
-            >
+                  volume={realtimeAudio.volume}
-              <VolumeMeter volume={realtimeAudio.volume} />
+                  isMuted={realtimeAudio.isMuted}
-            </Pressable>
+                  isDetecting={realtimeAudio.isDetecting}
+                  isSpeaking={realtimeAudio.isSpeaking}
+                />
+                {/* Debug timer */}
+                {(realtimeAudio.isDetecting || realtimeAudio.isSpeaking) && (
+                  <Text style={styles.debugTimer}>
+                    {(realtimeAudio.segmentDuration / 1000).toFixed(1)}s
+                  </Text>
+                )}
+              </View>
+              <View style={styles.realtimeModeButtons}>
+                {/* Mute button */}
+                <Pressable
+                  onPress={() => realtimeAudio.toggleMute()}
+                  style={[
+                    styles.realtimeMuteButton,
+                    realtimeAudio.isMuted && styles.realtimeMuteButtonActive,
+                  ]}
+                >
+                  <MicOff
+                    size={20}
+                    color={
+                      realtimeAudio.isMuted
+                        ? defaultTheme.colors.background
+                        : defaultTheme.colors.foreground
+                    }
+                  />
+                </Pressable>
+                {/* Close button */}
+                <Pressable
+                  onPress={handleRealtimeToggle}
+                  style={styles.realtimeCloseButton}
+                >
+                  <Square size={18} color="white" fill="white" />
+                </Pressable>
+              </View>
+            </View>
           ) : (
             // Normal mode - show text input and buttons
             <View style={styles.inputArea}>
@@ -1420,4 +1477,39 @@ const styles = StyleSheet.create((theme) => ({
   buttonDisabled: {
     opacity: theme.opacity[50],
   },
+  realtimeModeButtons: {
+    flexDirection: "row",
+    alignItems: "center",
+    justifyContent: "center",
+    gap: theme.spacing[3],
+    paddingTop: theme.spacing[4],
+  },
+  realtimeMuteButton: {
+    width: 48,
+    height: 48,
+    borderRadius: theme.borderRadius.full,
+    alignItems: "center",
+    justifyContent: "center",
+    backgroundColor: theme.colors.muted,
+    borderWidth: theme.borderWidth[2],
+    borderColor: theme.colors.border,
+  },
+  realtimeMuteButtonActive: {
+    backgroundColor: theme.colors.palette.red[500],
+    borderColor: theme.colors.palette.red[600],
+  },
+  realtimeCloseButton: {
+    width: 48,
+    height: 48,
+    borderRadius: theme.borderRadius.full,
+    alignItems: "center",
+    justifyContent: "center",
+    backgroundColor: theme.colors.palette.red[600],
+  },
+  debugTimer: {
+    marginTop: theme.spacing[2],
+    color: theme.colors.mutedForeground,
+    fontSize: theme.fontSize.sm,
+    fontFamily: "monospace",
+  },
 }));

packages/app/src/components/volume-meter.tsx

@@ -13,15 +13,18 @@ import { StyleSheet, useUnistyles } from "react-native-unistyles";
 
 interface VolumeMeterProps {
   volume: number;
+  isMuted?: boolean;
+  isDetecting?: boolean;
+  isSpeaking?: boolean;
 }
 
-export function VolumeMeter({ volume }: VolumeMeterProps) {
+export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSpeaking = false }: VolumeMeterProps) {
   const { theme } = useUnistyles();
 
   // Base dimensions
   const LINE_SPACING = 8;
-  const MAX_HEIGHT = 80;
+  const MAX_HEIGHT = 50;
-  const MIN_HEIGHT = 4;
+  const MIN_HEIGHT = 20;
 
   // Shared values for each line's height
   const line1Height = useSharedValue(MIN_HEIGHT);
@@ -35,6 +38,14 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
 
   // Start idle animations with different phases
   useEffect(() => {
+    if (isMuted) {
+      // When muted, set pulse to 1 (no animation)
+      line1Pulse.value = 1;
+      line2Pulse.value = 1;
+      line3Pulse.value = 1;
+      return;
+    }
+
     // Line 1 - fastest pulse
     line1Pulse.value = withRepeat(
       withSequence(
@@ -66,10 +77,18 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
       -1,
       false
     );
-  }, []);
+  }, [isMuted]);
 
   // Update heights based on volume with different responsiveness per line
   useEffect(() => {
+    if (isMuted) {
+      // When muted, keep lines at minimum height without animation
+      line1Height.value = MIN_HEIGHT;
+      line2Height.value = MIN_HEIGHT;
+      line3Height.value = MIN_HEIGHT;
+      return;
+    }
+
     if (volume > 0.01) {
       // Active volume - animate heights based on volume
       // Line 1 - most responsive, follows volume closely
@@ -107,23 +126,41 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
         stiffness: 150,
       });
     }
-  }, [volume]);
+  }, [volume, isMuted]);
 
   // Animated styles for each line
-  const line1Style = useAnimatedStyle(() => ({
+  const line1Style = useAnimatedStyle(() => {
-    height: line1Height.value * (volume > 0.01 ? 1 : line1Pulse.value),
+    const isActive = isDetecting || isSpeaking;
-    opacity: 0.8 + (volume * 0.2),
+    const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
-  }));
+    const volumeBoost = isMuted ? 0 : volume * 0.3;
+    return {
+      height: line1Height.value * (isMuted || volume > 0.01 ? 1 : line1Pulse.value),
+      opacity: baseOpacity + volumeBoost,
+    };
+  });
 
-  const line2Style = useAnimatedStyle(() => ({
+  const line2Style = useAnimatedStyle(() => {
-    height: line2Height.value * (volume > 0.01 ? 1 : line2Pulse.value),
+    const isActive = isDetecting || isSpeaking;
-    opacity: 0.7 + (volume * 0.3),
+    const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
-  }));
+    const volumeBoost = isMuted ? 0 : volume * 0.3;
+    return {
+      height: line2Height.value * (isMuted || volume > 0.01 ? 1 : line2Pulse.value),
+      opacity: baseOpacity + volumeBoost,
+    };
+  });
 
-  const line3Style = useAnimatedStyle(() => ({
+  const line3Style = useAnimatedStyle(() => {
-    height: line3Height.value * (volume > 0.01 ? 1 : line3Pulse.value),
+    const isActive = isDetecting || isSpeaking;
-    opacity: 0.6 + (volume * 0.4),
+    const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
-  }));
+    const volumeBoost = isMuted ? 0 : volume * 0.3;
+    return {
+      height: line3Height.value * (isMuted || volume > 0.01 ? 1 : line3Pulse.value),
+      opacity: baseOpacity + volumeBoost,
+    };
+  });
+
+  const lineColor = "#FFFFFF";
+  const lineWidth = 8;
 
   return (
     <View style={styles.container}>
@@ -131,8 +168,8 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
         style={[
           styles.line,
           {
-            width: 4,
+            width: lineWidth,
-            backgroundColor: theme.colors.palette.blue[400],
+            backgroundColor: lineColor,
           },
           line1Style,
         ]}
@@ -142,8 +179,8 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
         style={[
           styles.line,
           {
-            width: 6,
+            width: lineWidth,
-            backgroundColor: theme.colors.palette.blue[500],
+            backgroundColor: lineColor,
           },
           line2Style,
         ]}
@@ -153,8 +190,8 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
         style={[
           styles.line,
           {
-            width: 4,
+            width: lineWidth,
-            backgroundColor: theme.colors.palette.blue[400],
+            backgroundColor: lineColor,
           },
           line3Style,
         ]}

packages/app/src/hooks/use-speechmatics-audio.ts

@@ -22,10 +22,13 @@ export interface SpeechmaticsAudioConfig {
 export interface SpeechmaticsAudio {
   start: () => Promise<void>;
   stop: () => Promise<void>;
+  toggleMute: () => void;
   isActive: boolean;
   isSpeaking: boolean;
   isDetecting: boolean;
+  isMuted: boolean;
   volume: number;
+  segmentDuration: number;
 }
 
 /**
@@ -131,6 +134,8 @@ export function useSpeechmaticsAudio(
   const [isDetecting, setIsDetecting] = useState(false);
   const [audioInitialized, setAudioInitialized] = useState(false);
   const [volume, setVolume] = useState(0);
+  const [isMuted, setIsMuted] = useState(false);
+  const [segmentDuration, setSegmentDuration] = useState(0);
 
   const audioBufferRef = useRef<Uint8Array[]>([]);
   const silenceStartRef = useRef<number | null>(null);
@@ -164,12 +169,28 @@ export function useSpeechmaticsAudio(
     };
   }, []);
 
+  // Update segment duration timer
+  useEffect(() => {
+    if (!isDetecting && !isSpeaking) {
+      setSegmentDuration(0);
+      return;
+    }
+
+    const startTime = speechDetectionStartRef.current || Date.now();
+    const interval = setInterval(() => {
+      const elapsed = Date.now() - startTime;
+      setSegmentDuration(elapsed);
+    }, 100);
+
+    return () => clearInterval(interval);
+  }, [isDetecting, isSpeaking]);
+
   // Listen to microphone data
   useExpoTwoWayAudioEventListener(
     "onMicrophoneData",
     useCallback<MicrophoneDataCallback>(
       (event) => {
-        if (!isActive) return;
+        if (!isActive || isMuted) return;
 
         const pcmData: Uint8Array = event.data;
 
@@ -179,7 +200,7 @@ export function useSpeechmaticsAudio(
           audioBufferRef.current.push(pcmData);
         }
       },
-      [isActive]
+      [isActive, isMuted]
     )
   );
 
@@ -192,6 +213,9 @@ export function useSpeechmaticsAudio(
 
         const volumeLevel: number = event.data;
         setVolume(volumeLevel);
+
+        if (isMuted) return;
+
         const speechDetected = volumeLevel > VOLUME_THRESHOLD;
 
         // console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD);
@@ -285,7 +309,7 @@ export function useSpeechmaticsAudio(
           }
         }
       },
-      [isActive, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config]
+      [isActive, isMuted, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config]
     )
   );
 
@@ -333,10 +357,32 @@ export function useSpeechmaticsAudio(
     setIsSpeaking(false);
     setIsDetecting(false);
     setVolume(0);
+    setIsMuted(false);
 
     console.log("[SpeechmaticsAudio] Audio capture stopped");
   }
 
+  function toggleMute(): void {
+    setIsMuted((prev) => {
+      const newMuted = !prev;
+      console.log("[SpeechmaticsAudio] Mute toggled:", newMuted);
+
+      if (newMuted) {
+        // Clear any ongoing speech detection/speaking state
+        audioBufferRef.current = [];
+        isSpeakingRef.current = false;
+        speechConfirmedRef.current = false;
+        speechDetectionStartRef.current = null;
+        detectionSilenceStartRef.current = null;
+        silenceStartRef.current = null;
+        setIsSpeaking(false);
+        setIsDetecting(false);
+      }
+
+      return newMuted;
+    });
+  }
+
   // Cleanup on unmount
   useEffect(() => {
     return () => {
@@ -349,9 +395,12 @@ export function useSpeechmaticsAudio(
   return {
     start,
     stop,
+    toggleMute,
     isActive,
     isSpeaking,
     isDetecting,
+    isMuted,
     volume,
+    segmentDuration,
   };
 }

packages/server/src/server/agent/tts-manager.ts

@@ -26,7 +26,8 @@ export class TTSManager {
   public async generateAndWaitForPlayback(
     text: string,
     emitMessage: (msg: SessionOutboundMessage) => void,
-    abortSignal: AbortSignal
+    abortSignal: AbortSignal,
+    isRealtimeMode: boolean
   ): Promise<void> {
     if (abortSignal.aborted) {
       console.log(
@@ -78,13 +79,14 @@ export class TTSManager {
       });
     }
 
-    // Emit audio output message
+    // Emit audio output message (include mode for drift protection)
     emitMessage({
       type: "audio_output",
       payload: {
         id: audioId,
         audio: audio.toString("base64"),
         format,
+        isRealtimeMode,
       },
     });
 

packages/server/src/server/messages.ts

@@ -7,6 +7,7 @@ import { z } from "zod";
 export const UserTextMessageSchema = z.object({
   type: z.literal("user_text"),
   text: z.string(),
+  disableTTS: z.boolean().optional(),
 });
 
 export const AudioChunkMessageSchema = z.object({
@@ -39,6 +40,11 @@ export const DeleteConversationRequestMessageSchema = z.object({
   conversationId: z.string(),
 });
 
+export const SetRealtimeModeMessageSchema = z.object({
+  type: z.literal("set_realtime_mode"),
+  enabled: z.boolean(),
+});
+
 export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
   UserTextMessageSchema,
   AudioChunkMessageSchema,
@@ -47,6 +53,7 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
   LoadConversationRequestMessageSchema,
   ListConversationsRequestMessageSchema,
   DeleteConversationRequestMessageSchema,
+  SetRealtimeModeMessageSchema,
 ]);
 
 export type SessionInboundMessage = z.infer<typeof SessionInboundMessageSchema>;
@@ -88,6 +95,7 @@ export const AudioOutputMessageSchema = z.object({
     audio: z.string(), // base64 encoded
     format: z.string(),
     id: z.string(),
+    isRealtimeMode: z.boolean(), // Mode when audio was generated (for drift protection)
   }),
 });
 
@@ -164,6 +172,12 @@ export const AgentStatusMessageSchema = z.object({
       type: z.literal("claude"),
       sessionId: z.string().optional(),
       error: z.string().optional(),
+      currentModeId: z.string().optional(),
+      availableModes: z.array(z.object({
+        id: z.string(),
+        name: z.string(),
+        description: z.string().nullable().optional(),
+      })).optional(),
     }),
   }),
 });

packages/server/src/server/session.ts

@@ -54,6 +54,9 @@ export class Session {
   private processingPhase: ProcessingPhase = "idle";
   private currentStreamPromise: Promise<void> | null = null;
 
+  // Realtime mode state
+  private isRealtimeMode = false;
+
   // Audio buffering for interruption handling
   private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [];
   private bufferTimeout: NodeJS.Timeout | null = null;
@@ -165,12 +168,12 @@ export class Session {
                   info: {
                     id: info.id,
                     status: info.status,
-                    createdAt: info.createdAt.toISOString(),
+                    createdAt: info.createdAt,
                     type: info.type,
-                    sessionId: info.sessionId,
+                    sessionId: info.sessionId ?? undefined,
-                    error: info.error,
+                    error: info.error ?? undefined,
-                    currentModeId: info.currentModeId,
+                    currentModeId: info.currentModeId ?? undefined,
-                    availableModes: info.availableModes,
+                    availableModes: info.availableModes ?? undefined,
                   },
                 },
               });
@@ -300,6 +303,10 @@ export class Session {
         case "delete_conversation_request":
           await this.handleDeleteConversation(msg.conversationId);
           break;
+
+        case "set_realtime_mode":
+          this.handleSetRealtimeMode(msg.enabled);
+          break;
       }
     } catch (error: any) {
       console.error(
@@ -400,6 +407,16 @@ export class Session {
     }
   }
 
+  /**
+   * Handle realtime mode toggle
+   */
+  private handleSetRealtimeMode(enabled: boolean): void {
+    this.isRealtimeMode = enabled;
+    console.log(
+      `[Session ${this.clientId}] Realtime mode ${enabled ? "enabled" : "disabled"}`
+    );
+  }
+
   /**
    * Send current session state (live agents and commands) to client
    */
@@ -471,8 +488,8 @@ export class Session {
     // Add to conversation
     this.messages.push({ role: "user", content: text });
 
-    // Process through LLM (TTS enabled for text input)
+    // Process through LLM (TTS enabled in realtime mode for voice conversations)
-    this.currentStreamPromise = this.processWithLLM(true);
+    this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode);
     await this.currentStreamPromise;
   }
 
@@ -618,9 +635,9 @@ export class Session {
       // Add to conversation
       this.messages.push({ role: "user", content: result.text });
 
-      // Set phase to LLM and process
+      // Set phase to LLM and process (TTS enabled in realtime mode for voice conversations)
       this.setPhase("llm");
-      this.currentStreamPromise = this.processWithLLM(true); // Enable TTS for voice input
+      this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode);
       await this.currentStreamPromise;
       this.setPhase("idle");
     } catch (error: any) {
@@ -648,12 +665,14 @@ export class Session {
 
     const flushTextBuffer = () => {
       if (textBuffer.length > 0) {
-        // TTS handling
+        // TTS handling (capture mode at generation time for drift protection)
         if (enableTTS) {
+          const modeAtGeneration = this.isRealtimeMode;
           pendingTTS = this.ttsManager.generateAndWaitForPlayback(
             textBuffer,
             (msg) => this.emit(msg),
-            this.abortController.signal
+            this.abortController.signal,
+            modeAtGeneration
           );
         }