refactor: optimize audio initialization and fix agent interactions

- Lazy initialize audio (on-demand instead of mount) in both player and Speechmatics hooks
- Add audio session teardown on stop for proper cleanup
- Implement agent audio recording with sendAgentAudio in AgentInputArea
- Fix Android navigation race condition in CreateAgentModal (navigate before closing)
- Add animations to global footer and agent screen (FadeIn/FadeOut)
- Auto-cancel stuck agents when sending new messages
- Fix permission request ID propagation from AgentManager to Session
- Apply theme background color via AppContainer wrapper

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

packages/app/src/app/_layout.tsx

@@ -8,6 +8,17 @@ import { RealtimeProvider } from "@/contexts/realtime-context";
 import { useSettings } from "@/hooks/use-settings";
 import { View, ActivityIndicator } from "react-native";
 import { GlobalFooter } from "@/components/global-footer";
+import { useUnistyles } from "react-native-unistyles";
+
+function AppContainer({ children }: { children: React.ReactNode }) {
+  const { theme } = useUnistyles();
+
+  return (
+    <View style={{ flex: 1, backgroundColor: theme.colors.background }}>
+      {children}
+    </View>
+  );
+}
 
 function ProvidersWrapper({ children }: { children: React.ReactNode }) {
   const { settings, isLoading } = useSettings();
@@ -41,7 +52,7 @@ export default function RootLayout() {
         <KeyboardProvider>
           <BottomSheetModalProvider>
             <ProvidersWrapper>
-              <View style={{ flex: 1 }}>
+              <AppContainer>
                 <Stack
                   screenOptions={{
                     headerShown: false,
@@ -60,7 +71,7 @@ export default function RootLayout() {
                   <Stack.Screen name="audio-test" />
                 </Stack>
                 <GlobalFooter />
-              </View>
+              </AppContainer>
             </ProvidersWrapper>
           </BottomSheetModalProvider>
         </KeyboardProvider>

packages/app/src/app/agent/[id].tsx

@@ -2,7 +2,7 @@ import { View, Text } from "react-native";
 import { useLocalSearchParams } from "expo-router";
 import { useSafeAreaInsets } from "react-native-safe-area-context";
 import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
-import ReanimatedAnimated, { useAnimatedStyle } from "react-native-reanimated";
+import ReanimatedAnimated, { useAnimatedStyle, FadeIn } from "react-native-reanimated";
 import { StyleSheet, useUnistyles } from "react-native-unistyles";
 import { BackHeader } from "@/components/headers/back-header";
 import { AgentStreamView } from "@/components/agent-stream-view";
@@ -61,9 +61,12 @@ export default function AgentScreen() {
         />
 
         {/* Agent Input - will hide itself when realtime is active */}
-        <View style={[styles.inputContainer, { paddingBottom: insets.bottom }]}>
+        <ReanimatedAnimated.View
+          entering={FadeIn.duration(250).delay(100)}
+          style={[styles.inputContainer, { paddingBottom: insets.bottom }]}
+        >
           <AgentInputArea agentId={id!} />
-        </View>
+        </ReanimatedAnimated.View>
       </ReanimatedAnimated.View>
     </View>
   );

packages/app/src/components/agent-input-area.tsx

@@ -13,7 +13,7 @@ interface AgentInputAreaProps {
 
 export function AgentInputArea({ agentId }: AgentInputAreaProps) {
   const { theme } = useUnistyles();
-  const { agents, ws, sendAgentMessage, setAgentMode } = useSession();
+  const { agents, ws, sendAgentMessage, sendAgentAudio, setAgentMode } = useSession();
   const { isRealtimeMode, startRealtime } = useRealtime();
   const audioRecorder = useAudioRecorder();
 
@@ -54,10 +54,17 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
 
         if (audioData) {
           setIsProcessing(true);
-          // TODO: Send audio to agent
-          // For now, just log it
           console.log("[AgentInput] Audio recorded:", audioData.size, "bytes");
-          setIsProcessing(false);
+
+          try {
+            // Send audio to agent for transcription and processing
+            await sendAgentAudio(agentId, audioData);
+            console.log("[AgentInput] Audio sent to agent");
+          } catch (error) {
+            console.error("[AgentInput] Failed to send audio:", error);
+          } finally {
+            setIsProcessing(false);
+          }
         }
       } catch (error) {
         console.error("[AgentInput] Failed to stop recording:", error);

packages/app/src/components/create-agent-modal.tsx

@@ -131,10 +131,14 @@ export function CreateAgentModal({
         console.log("[CreateAgentModal] Agent created:", agentId);
         setIsLoading(false);
         setPendingRequestId(null);
-        handleClose();
 
-        // Navigate to the agent page
+        // Navigate to the agent page BEFORE closing modal
+        // This prevents race condition on Android where router.push() happens
+        // while the modal is unmounting, causing NullPointerException
         router.push(`/agent/${agentId}`);
+
+        // Close modal after navigation starts
+        handleClose();
       }
     });
 

packages/app/src/components/global-footer.tsx

@@ -1,42 +1,55 @@
 import { View, Pressable } from "react-native";
-import { usePathname, useLocalSearchParams } from "expo-router";
+import { usePathname } from "expo-router";
 import { useSafeAreaInsets } from "react-native-safe-area-context";
 import { StyleSheet, useUnistyles } from "react-native-unistyles";
 import { AudioLines } from "lucide-react-native";
 import { useRealtime } from "@/contexts/realtime-context";
 import { useSession } from "@/contexts/session-context";
 import { RealtimeControls } from "./realtime-controls";
+import Animated, { FadeIn, FadeOut } from "react-native-reanimated";
 
 export function GlobalFooter() {
   const { theme } = useUnistyles();
   const insets = useSafeAreaInsets();
   const pathname = usePathname();
-  const params = useLocalSearchParams();
   const { isRealtimeMode, startRealtime } = useRealtime();
   const { ws } = useSession();
 
   // Determine current screen type
   const isAgentScreen = pathname?.startsWith("/agent/");
-  const isOrchestratorScreen = pathname === "/orchestrator";
-  const isHomeScreen = pathname === "/";
 
-  // If realtime is active, always show realtime controls
+  // Determine if footer should be visible
+  // Hidden when: on agent screen AND realtime is off
+  const shouldHide = isAgentScreen && !isRealtimeMode;
+
+  // If realtime is active, show realtime controls
   if (isRealtimeMode) {
     return (
-      <View style={[styles.container, { paddingBottom: insets.bottom }]}>
+      <Animated.View
+        entering={FadeIn.duration(400)}
+        exiting={FadeOut.duration(250)}
+        style={[styles.container, { paddingBottom: insets.bottom }]}
+      >
         <RealtimeControls />
-      </View>
+      </Animated.View>
     );
   }
 
-  // If on agent screen and realtime is off, hide footer (agent has its own controls)
+  // For home and orchestrator screens, show centered realtime button
-  if (isAgentScreen) {
+  // On agent screens, don't render at all
+  if (shouldHide) {
     return null;
   }
 
-  // For home and orchestrator screens, show centered realtime button
   return (
-    <View style={[styles.container, { paddingBottom: insets.bottom }]}>
+    <Animated.View
+      entering={FadeIn.duration(250)}
+      exiting={FadeOut.duration(250)}
+      style={[
+        styles.container,
+        { paddingBottom: insets.bottom }
+      ]}
+    >
       <View style={styles.centeredButtonContainer}>
         <Pressable
           onPress={startRealtime}
@@ -49,7 +62,7 @@ export function GlobalFooter() {
           <AudioLines size={24} color="white" />
         </Pressable>
       </View>
-    </View>
+    </Animated.View>
   );
 }
 

packages/app/src/contexts/session-context.tsx

@@ -120,6 +120,7 @@ interface SessionContextValue {
 
   // Helpers
   sendAgentMessage: (agentId: string, message: string) => void;
+  sendAgentAudio: (agentId: string, audioBlob: Blob) => Promise<void>;
   createAgent: (options: { cwd: string; initialMode?: string; requestId?: string }) => void;
   setAgentMode: (agentId: string, modeId: string) => void;
   respondToPermission: (requestId: string, agentId: string, sessionId: string, selectedOptionIds: string[]) => void;
@@ -516,6 +517,40 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
     ws.send(msg);
   }, [ws]);
 
+  const sendAgentAudio = useCallback(async (agentId: string, audioBlob: Blob) => {
+    try {
+      // Convert blob to base64
+      const arrayBuffer = await audioBlob.arrayBuffer();
+      const bytes = new Uint8Array(arrayBuffer);
+      let binary = '';
+      for (let i = 0; i < bytes.length; i++) {
+        binary += String.fromCharCode(bytes[i]);
+      }
+      const base64Audio = btoa(binary);
+
+      // Determine format from MIME type
+      const format = audioBlob.type.split('/')[1] || 'm4a';
+
+      // Send audio message
+      const msg: WSInboundMessage = {
+        type: "session",
+        message: {
+          type: "send_agent_audio",
+          agentId,
+          audio: base64Audio,
+          format,
+          isLast: true,
+        },
+      };
+      ws.send(msg);
+
+      console.log("[Session] Sent audio to agent:", agentId, format, audioBlob.size, "bytes");
+    } catch (error) {
+      console.error("[Session] Failed to send audio:", error);
+      throw error;
+    }
+  }, [ws]);
+
   const createAgent = useCallback((options: { cwd: string; initialMode?: string; requestId?: string }) => {
     const msg: WSInboundMessage = {
       type: "session",
@@ -584,6 +619,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
     pendingPermissions,
     setPendingPermissions,
     sendAgentMessage,
+    sendAgentAudio,
     createAgent,
     setAgentMode,
     respondToPermission,

packages/app/src/hooks/use-audio-player.ts

@@ -1,4 +1,4 @@
-import { useState, useRef, useEffect } from 'react';
+import { useState, useRef } from 'react';
 import { initialize, playPCMData } from '@speechmatics/expo-two-way-audio';
 
 interface QueuedAudio {
@@ -45,20 +45,6 @@ export function useAudioPlayer() {
   const queueRef = useRef<QueuedAudio[]>([]);
   const isProcessingQueueRef = useRef(false);
 
-  // Initialize Speechmatics audio on mount
-  useEffect(() => {
-    const initializeAudio = async () => {
-      try {
-        await initialize();
-        setAudioInitialized(true);
-        console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)');
-      } catch (error) {
-        console.error('[AudioPlayer] Failed to initialize audio:', error);
-      }
-    };
-    initializeAudio();
-  }, []);
-
   async function play(audioData: Blob): Promise<number> {
     return new Promise((resolve, reject) => {
       // Add to queue with its promise handlers
@@ -112,8 +98,12 @@ export function useAudioPlayer() {
           `[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
         );
 
+        // Initialize audio if not already initialized
         if (!audioInitialized) {
-          throw new Error('Audio not initialized');
+          console.log('[AudioPlayer] Initializing audio...');
+          await initialize();
+          setAudioInitialized(true);
+          console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)');
         }
 
         // Get PCM data from blob (server now sends PCM format)

packages/app/src/hooks/use-speechmatics-audio.ts

@@ -149,26 +149,6 @@ export function useSpeechmaticsAudio(
   const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration;
   const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod;
 
-  // Initialize audio on mount
-  useEffect(() => {
-    const initializeAudio = async () => {
-      try {
-        await initialize();
-        setAudioInitialized(true);
-        console.log("[SpeechmaticsAudio] Audio initialized");
-      } catch (error) {
-        console.error("[SpeechmaticsAudio] Initialization error:", error);
-        const err = error instanceof Error ? error : new Error(String(error));
-        config.onError?.(err);
-      }
-    };
-    initializeAudio();
-
-    return () => {
-      tearDown();
-    };
-  }, []);
-
   // Update segment duration timer
   useEffect(() => {
     if (!isDetecting && !isSpeaking) {
@@ -314,16 +294,20 @@ export function useSpeechmaticsAudio(
   );
 
   async function start(): Promise<void> {
-    if (!audioInitialized) {
-      throw new Error("Audio not initialized");
-    }
-
     if (isActive) {
       console.log("[SpeechmaticsAudio] Already active");
       return;
     }
 
     try {
+      // Initialize audio if not already initialized
+      if (!audioInitialized) {
+        console.log("[SpeechmaticsAudio] Initializing audio...");
+        await initialize();
+        setAudioInitialized(true);
+        console.log("[SpeechmaticsAudio] Audio initialized");
+      }
+
       console.log("[SpeechmaticsAudio] Starting audio capture...");
 
       // Start recording
@@ -344,7 +328,16 @@ export function useSpeechmaticsAudio(
     console.log("[SpeechmaticsAudio] Stopping audio capture...");
 
     // Stop recording
-    toggleRecording(false);
+    if (isActive) {
+      toggleRecording(false);
+    }
+
+    // Tear down audio session
+    if (audioInitialized) {
+      tearDown();
+      setAudioInitialized(false);
+      console.log("[SpeechmaticsAudio] Audio torn down");
+    }
 
     // Reset state
     audioBufferRef.current = [];
@@ -383,15 +376,6 @@ export function useSpeechmaticsAudio(
     });
   }
 
-  // Cleanup on unmount
-  useEffect(() => {
-    return () => {
-      if (isActive) {
-        stop();
-      }
-    };
-  }, [isActive]);
-
   return {
     start,
     stop,

packages/server/src/server/acp/agent-manager.ts

@@ -268,6 +268,20 @@ export class AgentManager {
       throw new Error(`Agent ${agentId} is ${agent.status}`);
     }
 
+    // Auto-cancel if agent is currently processing
+    // This allows users to interrupt stuck agents by sending a new message
+    if (agent.status === "processing") {
+      console.log(`[Agent ${agentId}] Auto-cancelling current task before new prompt`);
+      try {
+        await this.cancelAgent(agentId);
+        // Small delay to allow cancellation to propagate
+        await new Promise(resolve => setTimeout(resolve, 100));
+      } catch (error) {
+        console.warn(`[Agent ${agentId}] Cancel failed, continuing with new prompt:`, error);
+        // Continue anyway - the new prompt might still work
+      }
+    }
+
     // Reset message IDs for new turn - ensures each prompt gets fresh responses
     agent.currentAssistantMessageId = null;
     agent.currentThoughtId = null;
@@ -784,6 +798,7 @@ export class AgentManager {
         timestamp: new Date(),
         notification: {
           type: 'permission',
+          requestId,  // Pass the requestId that we stored in pendingPermissions
           request: params,
         },
       };

packages/server/src/server/acp/types.ts

@@ -23,7 +23,7 @@ export interface EnrichedSessionNotification extends Omit<SessionNotification, '
  */
 export type AgentNotification =
   | { type: 'session'; notification: EnrichedSessionNotification }
-  | { type: 'permission'; request: RequestPermissionRequest }
+  | { type: 'permission'; requestId: string; request: RequestPermissionRequest }
   | { type: 'status'; status: AgentStatus; error?: string };
 
 /**

packages/server/src/server/session.ts

@@ -178,14 +178,14 @@ export class Session {
             type: "agent_permission_request",
             payload: {
               agentId,
-              requestId: uuidv4(), // Generate request ID
+              requestId: notification.requestId, // Use the requestId from AgentManager
               sessionId: permissionRequest.sessionId,
               toolCall: permissionRequest.toolCall,
               options: permissionRequest.options,
             },
           });
           console.log(
-            `[Session ${this.clientId}] Forwarded permission request for agent ${agentId}`
+            `[Session ${this.clientId}] Forwarded permission request for agent ${agentId} with requestId ${notification.requestId}`
           );
           return;
         }