diff --git a/packages/app/src/app/_layout.tsx b/packages/app/src/app/_layout.tsx index 982cc17af..983456c98 100644 --- a/packages/app/src/app/_layout.tsx +++ b/packages/app/src/app/_layout.tsx @@ -8,6 +8,17 @@ import { RealtimeProvider } from "@/contexts/realtime-context"; import { useSettings } from "@/hooks/use-settings"; import { View, ActivityIndicator } from "react-native"; import { GlobalFooter } from "@/components/global-footer"; +import { useUnistyles } from "react-native-unistyles"; + +function AppContainer({ children }: { children: React.ReactNode }) { + const { theme } = useUnistyles(); + + return ( + + {children} + + ); +} function ProvidersWrapper({ children }: { children: React.ReactNode }) { const { settings, isLoading } = useSettings(); @@ -41,7 +52,7 @@ export default function RootLayout() { - + - + diff --git a/packages/app/src/app/agent/[id].tsx b/packages/app/src/app/agent/[id].tsx index 7d26106be..656d118cf 100644 --- a/packages/app/src/app/agent/[id].tsx +++ b/packages/app/src/app/agent/[id].tsx @@ -2,7 +2,7 @@ import { View, Text } from "react-native"; import { useLocalSearchParams } from "expo-router"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; -import ReanimatedAnimated, { useAnimatedStyle } from "react-native-reanimated"; +import ReanimatedAnimated, { useAnimatedStyle, FadeIn } from "react-native-reanimated"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; import { BackHeader } from "@/components/headers/back-header"; import { AgentStreamView } from "@/components/agent-stream-view"; @@ -61,9 +61,12 @@ export default function AgentScreen() { /> {/* Agent Input - will hide itself when realtime is active */} - + - + ); diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx index bf07025fc..48d09cb72 100644 --- a/packages/app/src/components/agent-input-area.tsx +++ b/packages/app/src/components/agent-input-area.tsx @@ -13,7 +13,7 @@ interface AgentInputAreaProps { export function AgentInputArea({ agentId }: AgentInputAreaProps) { const { theme } = useUnistyles(); - const { agents, ws, sendAgentMessage, setAgentMode } = useSession(); + const { agents, ws, sendAgentMessage, sendAgentAudio, setAgentMode } = useSession(); const { isRealtimeMode, startRealtime } = useRealtime(); const audioRecorder = useAudioRecorder(); @@ -54,10 +54,17 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { if (audioData) { setIsProcessing(true); - // TODO: Send audio to agent - // For now, just log it console.log("[AgentInput] Audio recorded:", audioData.size, "bytes"); - setIsProcessing(false); + + try { + // Send audio to agent for transcription and processing + await sendAgentAudio(agentId, audioData); + console.log("[AgentInput] Audio sent to agent"); + } catch (error) { + console.error("[AgentInput] Failed to send audio:", error); + } finally { + setIsProcessing(false); + } } } catch (error) { console.error("[AgentInput] Failed to stop recording:", error); diff --git a/packages/app/src/components/create-agent-modal.tsx b/packages/app/src/components/create-agent-modal.tsx index 2fc69fed4..ef31660e2 100644 --- a/packages/app/src/components/create-agent-modal.tsx +++ b/packages/app/src/components/create-agent-modal.tsx @@ -131,10 +131,14 @@ export function CreateAgentModal({ console.log("[CreateAgentModal] Agent created:", agentId); setIsLoading(false); setPendingRequestId(null); - handleClose(); - // Navigate to the agent page + // Navigate to the agent page BEFORE closing modal + // This prevents race condition on Android where router.push() happens + // while the modal is unmounting, causing NullPointerException router.push(`/agent/${agentId}`); + + // Close modal after navigation starts + handleClose(); } }); diff --git a/packages/app/src/components/global-footer.tsx b/packages/app/src/components/global-footer.tsx index 9b158513c..0a774a337 100644 --- a/packages/app/src/components/global-footer.tsx +++ b/packages/app/src/components/global-footer.tsx @@ -1,42 +1,55 @@ import { View, Pressable } from "react-native"; -import { usePathname, useLocalSearchParams } from "expo-router"; +import { usePathname } from "expo-router"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; import { AudioLines } from "lucide-react-native"; import { useRealtime } from "@/contexts/realtime-context"; import { useSession } from "@/contexts/session-context"; import { RealtimeControls } from "./realtime-controls"; +import Animated, { FadeIn, FadeOut } from "react-native-reanimated"; export function GlobalFooter() { const { theme } = useUnistyles(); const insets = useSafeAreaInsets(); const pathname = usePathname(); - const params = useLocalSearchParams(); const { isRealtimeMode, startRealtime } = useRealtime(); const { ws } = useSession(); // Determine current screen type const isAgentScreen = pathname?.startsWith("/agent/"); - const isOrchestratorScreen = pathname === "/orchestrator"; - const isHomeScreen = pathname === "/"; - // If realtime is active, always show realtime controls + // Determine if footer should be visible + // Hidden when: on agent screen AND realtime is off + const shouldHide = isAgentScreen && !isRealtimeMode; + + // If realtime is active, show realtime controls if (isRealtimeMode) { return ( - + - + ); } - // If on agent screen and realtime is off, hide footer (agent has its own controls) - if (isAgentScreen) { + // For home and orchestrator screens, show centered realtime button + // On agent screens, don't render at all + if (shouldHide) { return null; } - // For home and orchestrator screens, show centered realtime button return ( - + - + ); } diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx index 603deb051..246b1f850 100644 --- a/packages/app/src/contexts/session-context.tsx +++ b/packages/app/src/contexts/session-context.tsx @@ -120,6 +120,7 @@ interface SessionContextValue { // Helpers sendAgentMessage: (agentId: string, message: string) => void; + sendAgentAudio: (agentId: string, audioBlob: Blob) => Promise; createAgent: (options: { cwd: string; initialMode?: string; requestId?: string }) => void; setAgentMode: (agentId: string, modeId: string) => void; respondToPermission: (requestId: string, agentId: string, sessionId: string, selectedOptionIds: string[]) => void; @@ -516,6 +517,40 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { ws.send(msg); }, [ws]); + const sendAgentAudio = useCallback(async (agentId: string, audioBlob: Blob) => { + try { + // Convert blob to base64 + const arrayBuffer = await audioBlob.arrayBuffer(); + const bytes = new Uint8Array(arrayBuffer); + let binary = ''; + for (let i = 0; i < bytes.length; i++) { + binary += String.fromCharCode(bytes[i]); + } + const base64Audio = btoa(binary); + + // Determine format from MIME type + const format = audioBlob.type.split('/')[1] || 'm4a'; + + // Send audio message + const msg: WSInboundMessage = { + type: "session", + message: { + type: "send_agent_audio", + agentId, + audio: base64Audio, + format, + isLast: true, + }, + }; + ws.send(msg); + + console.log("[Session] Sent audio to agent:", agentId, format, audioBlob.size, "bytes"); + } catch (error) { + console.error("[Session] Failed to send audio:", error); + throw error; + } + }, [ws]); + const createAgent = useCallback((options: { cwd: string; initialMode?: string; requestId?: string }) => { const msg: WSInboundMessage = { type: "session", @@ -584,6 +619,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { pendingPermissions, setPendingPermissions, sendAgentMessage, + sendAgentAudio, createAgent, setAgentMode, respondToPermission, diff --git a/packages/app/src/hooks/use-audio-player.ts b/packages/app/src/hooks/use-audio-player.ts index 19b47990a..aea818453 100644 --- a/packages/app/src/hooks/use-audio-player.ts +++ b/packages/app/src/hooks/use-audio-player.ts @@ -1,4 +1,4 @@ -import { useState, useRef, useEffect } from 'react'; +import { useState, useRef } from 'react'; import { initialize, playPCMData } from '@speechmatics/expo-two-way-audio'; interface QueuedAudio { @@ -45,20 +45,6 @@ export function useAudioPlayer() { const queueRef = useRef([]); const isProcessingQueueRef = useRef(false); - // Initialize Speechmatics audio on mount - useEffect(() => { - const initializeAudio = async () => { - try { - await initialize(); - setAudioInitialized(true); - console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)'); - } catch (error) { - console.error('[AudioPlayer] Failed to initialize audio:', error); - } - }; - initializeAudio(); - }, []); - async function play(audioData: Blob): Promise { return new Promise((resolve, reject) => { // Add to queue with its promise handlers @@ -112,8 +98,12 @@ export function useAudioPlayer() { `[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})` ); + // Initialize audio if not already initialized if (!audioInitialized) { - throw new Error('Audio not initialized'); + console.log('[AudioPlayer] Initializing audio...'); + await initialize(); + setAudioInitialized(true); + console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)'); } // Get PCM data from blob (server now sends PCM format) diff --git a/packages/app/src/hooks/use-speechmatics-audio.ts b/packages/app/src/hooks/use-speechmatics-audio.ts index 52037b919..56b9eeeb2 100644 --- a/packages/app/src/hooks/use-speechmatics-audio.ts +++ b/packages/app/src/hooks/use-speechmatics-audio.ts @@ -149,26 +149,6 @@ export function useSpeechmaticsAudio( const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration; const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod; - // Initialize audio on mount - useEffect(() => { - const initializeAudio = async () => { - try { - await initialize(); - setAudioInitialized(true); - console.log("[SpeechmaticsAudio] Audio initialized"); - } catch (error) { - console.error("[SpeechmaticsAudio] Initialization error:", error); - const err = error instanceof Error ? error : new Error(String(error)); - config.onError?.(err); - } - }; - initializeAudio(); - - return () => { - tearDown(); - }; - }, []); - // Update segment duration timer useEffect(() => { if (!isDetecting && !isSpeaking) { @@ -314,16 +294,20 @@ export function useSpeechmaticsAudio( ); async function start(): Promise { - if (!audioInitialized) { - throw new Error("Audio not initialized"); - } - if (isActive) { console.log("[SpeechmaticsAudio] Already active"); return; } try { + // Initialize audio if not already initialized + if (!audioInitialized) { + console.log("[SpeechmaticsAudio] Initializing audio..."); + await initialize(); + setAudioInitialized(true); + console.log("[SpeechmaticsAudio] Audio initialized"); + } + console.log("[SpeechmaticsAudio] Starting audio capture..."); // Start recording @@ -344,7 +328,16 @@ export function useSpeechmaticsAudio( console.log("[SpeechmaticsAudio] Stopping audio capture..."); // Stop recording - toggleRecording(false); + if (isActive) { + toggleRecording(false); + } + + // Tear down audio session + if (audioInitialized) { + tearDown(); + setAudioInitialized(false); + console.log("[SpeechmaticsAudio] Audio torn down"); + } // Reset state audioBufferRef.current = []; @@ -383,15 +376,6 @@ export function useSpeechmaticsAudio( }); } - // Cleanup on unmount - useEffect(() => { - return () => { - if (isActive) { - stop(); - } - }; - }, [isActive]); - return { start, stop, diff --git a/packages/server/src/server/acp/agent-manager.ts b/packages/server/src/server/acp/agent-manager.ts index fa531c9f7..ef0d5eeb7 100644 --- a/packages/server/src/server/acp/agent-manager.ts +++ b/packages/server/src/server/acp/agent-manager.ts @@ -268,6 +268,20 @@ export class AgentManager { throw new Error(`Agent ${agentId} is ${agent.status}`); } + // Auto-cancel if agent is currently processing + // This allows users to interrupt stuck agents by sending a new message + if (agent.status === "processing") { + console.log(`[Agent ${agentId}] Auto-cancelling current task before new prompt`); + try { + await this.cancelAgent(agentId); + // Small delay to allow cancellation to propagate + await new Promise(resolve => setTimeout(resolve, 100)); + } catch (error) { + console.warn(`[Agent ${agentId}] Cancel failed, continuing with new prompt:`, error); + // Continue anyway - the new prompt might still work + } + } + // Reset message IDs for new turn - ensures each prompt gets fresh responses agent.currentAssistantMessageId = null; agent.currentThoughtId = null; @@ -784,6 +798,7 @@ export class AgentManager { timestamp: new Date(), notification: { type: 'permission', + requestId, // Pass the requestId that we stored in pendingPermissions request: params, }, }; diff --git a/packages/server/src/server/acp/types.ts b/packages/server/src/server/acp/types.ts index 37d981012..bc68b5b6b 100644 --- a/packages/server/src/server/acp/types.ts +++ b/packages/server/src/server/acp/types.ts @@ -23,7 +23,7 @@ export interface EnrichedSessionNotification extends Omit