jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: optimize audio initialization and fix agent interactions

Browse Source 
- Lazy initialize audio (on-demand instead of mount) in both player and Speechmatics hooks
- Add audio session teardown on stop for proper cleanup
- Implement agent audio recording with sendAgentAudio in AgentInputArea
- Fix Android navigation race condition in CreateAgentModal (navigate before closing)
- Add animations to global footer and agent screen (FadeIn/FadeOut)
- Auto-cancel stuck agents when sending new messages
- Fix permission request ID propagation from AgentManager to Session
- Apply theme background color via AppContainer wrapper

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Mohamed Boudra
2025-10-23 13:15:18 +02:00
co-authored by Claude
parent 10189514dd
commit 74fffcd8a3
11 changed files with 139 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/app/_layout.tsx
+13 -2
View File
@@ -8,6 +8,17 @@ import { RealtimeProvider } from "@/contexts/realtime-context";
import { useSettings } from "@/hooks/use-settings";
import { View, ActivityIndicator } from "react-native";
import { GlobalFooter } from "@/components/global-footer";
import { useUnistyles } from "react-native-unistyles";
function AppContainer({ children }: { children: React.ReactNode }) {
const { theme } = useUnistyles();
return (
<View style={{ flex: 1, backgroundColor: theme.colors.background }}>
{children}
</View>
);
}
function ProvidersWrapper({ children }: { children: React.ReactNode }) {
const { settings, isLoading } = useSettings();
@@ -41,7 +52,7 @@ export default function RootLayout() {
<KeyboardProvider>
<BottomSheetModalProvider>
<ProvidersWrapper>
<View style={{ flex: 1 }}>
<AppContainer>
<Stack
screenOptions={{
headerShown: false,
@@ -60,7 +71,7 @@ export default function RootLayout() {
<Stack.Screen name="audio-test" />
</Stack>
<GlobalFooter />
</View>
</AppContainer>
</ProvidersWrapper>
</BottomSheetModalProvider>
</KeyboardProvider>
packages/app/src/app/agent/[id].tsx
+6 -3
View File
@@ -2,7 +2,7 @@ import { View, Text } from "react-native";
import { useLocalSearchParams } from "expo-router";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
import ReanimatedAnimated, { useAnimatedStyle } from "react-native-reanimated";
import ReanimatedAnimated, { useAnimatedStyle, FadeIn } from "react-native-reanimated";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { BackHeader } from "@/components/headers/back-header";
import { AgentStreamView } from "@/components/agent-stream-view";
@@ -61,9 +61,12 @@ export default function AgentScreen() {
/>
{/* Agent Input - will hide itself when realtime is active */}
<View style={[styles.inputContainer, { paddingBottom: insets.bottom }]}>
<ReanimatedAnimated.View
entering={FadeIn.duration(250).delay(100)}
style={[styles.inputContainer, { paddingBottom: insets.bottom }]}
>
<AgentInputArea agentId={id!} />
</View>
</ReanimatedAnimated.View>
</ReanimatedAnimated.View>
</View>
);
packages/app/src/components/agent-input-area.tsx
+11 -4
View File
@@ -13,7 +13,7 @@ interface AgentInputAreaProps {
export function AgentInputArea({ agentId }: AgentInputAreaProps) {
const { theme } = useUnistyles();
const { agents, ws, sendAgentMessage, setAgentMode } = useSession();
const { agents, ws, sendAgentMessage, sendAgentAudio, setAgentMode } = useSession();
const { isRealtimeMode, startRealtime } = useRealtime();
const audioRecorder = useAudioRecorder();
@@ -54,10 +54,17 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
if (audioData) {
setIsProcessing(true);
// TODO: Send audio to agent
// For now, just log it
console.log("[AgentInput] Audio recorded:", audioData.size, "bytes");
setIsProcessing(false);
try {
// Send audio to agent for transcription and processing
await sendAgentAudio(agentId, audioData);
console.log("[AgentInput] Audio sent to agent");
} catch (error) {
console.error("[AgentInput] Failed to send audio:", error);
} finally {
setIsProcessing(false);
}
}
} catch (error) {
console.error("[AgentInput] Failed to stop recording:", error);
packages/app/src/components/create-agent-modal.tsx
+6 -2
View File
@@ -131,10 +131,14 @@ export function CreateAgentModal({
console.log("[CreateAgentModal] Agent created:", agentId);
setIsLoading(false);
setPendingRequestId(null);
handleClose();
// Navigate to the agent page
// Navigate to the agent page BEFORE closing modal
// This prevents race condition on Android where router.push() happens
// while the modal is unmounting, causing NullPointerException
router.push(`/agent/${agentId}`);
// Close modal after navigation starts
handleClose();
}
});
packages/app/src/components/global-footer.tsx
+25 -12
View File
@@ -1,42 +1,55 @@
import { View, Pressable } from "react-native";
import { usePathname, useLocalSearchParams } from "expo-router";
import { usePathname } from "expo-router";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { AudioLines } from "lucide-react-native";
import { useRealtime } from "@/contexts/realtime-context";
import { useSession } from "@/contexts/session-context";
import { RealtimeControls } from "./realtime-controls";
import Animated, { FadeIn, FadeOut } from "react-native-reanimated";
export function GlobalFooter() {
const { theme } = useUnistyles();
const insets = useSafeAreaInsets();
const pathname = usePathname();
const params = useLocalSearchParams();
const { isRealtimeMode, startRealtime } = useRealtime();
const { ws } = useSession();
// Determine current screen type
const isAgentScreen = pathname?.startsWith("/agent/");
const isOrchestratorScreen = pathname === "/orchestrator";
const isHomeScreen = pathname === "/";
// If realtime is active, always show realtime controls
// Determine if footer should be visible
// Hidden when: on agent screen AND realtime is off
const shouldHide = isAgentScreen && !isRealtimeMode;
// If realtime is active, show realtime controls
if (isRealtimeMode) {
return (
<View style={[styles.container, { paddingBottom: insets.bottom }]}>
<Animated.View
entering={FadeIn.duration(400)}
exiting={FadeOut.duration(250)}
style={[styles.container, { paddingBottom: insets.bottom }]}
>
<RealtimeControls />
</View>
</Animated.View>
);
}
// If on agent screen and realtime is off, hide footer (agent has its own controls)
if (isAgentScreen) {
// For home and orchestrator screens, show centered realtime button
// On agent screens, don't render at all
if (shouldHide) {
return null;
}
// For home and orchestrator screens, show centered realtime button
return (
<View style={[styles.container, { paddingBottom: insets.bottom }]}>
<Animated.View
entering={FadeIn.duration(250)}
exiting={FadeOut.duration(250)}
style={[
styles.container,
{ paddingBottom: insets.bottom }
]}
>
<View style={styles.centeredButtonContainer}>
<Pressable
onPress={startRealtime}
@@ -49,7 +62,7 @@ export function GlobalFooter() {
<AudioLines size={24} color="white" />
</Pressable>
</View>
</View>
</Animated.View>
);
}
packages/app/src/contexts/session-context.tsx
+36
View File
@@ -120,6 +120,7 @@ interface SessionContextValue {
// Helpers
sendAgentMessage: (agentId: string, message: string) => void;
sendAgentAudio: (agentId: string, audioBlob: Blob) => Promise<void>;
createAgent: (options: { cwd: string; initialMode?: string; requestId?: string }) => void;
setAgentMode: (agentId: string, modeId: string) => void;
respondToPermission: (requestId: string, agentId: string, sessionId: string, selectedOptionIds: string[]) => void;
@@ -516,6 +517,40 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
ws.send(msg);
}, [ws]);
const sendAgentAudio = useCallback(async (agentId: string, audioBlob: Blob) => {
try {
// Convert blob to base64
const arrayBuffer = await audioBlob.arrayBuffer();
const bytes = new Uint8Array(arrayBuffer);
let binary = '';
for (let i = 0; i < bytes.length; i++) {
binary += String.fromCharCode(bytes[i]);
}
const base64Audio = btoa(binary);
// Determine format from MIME type
const format = audioBlob.type.split('/')[1] || 'm4a';
// Send audio message
const msg: WSInboundMessage = {
type: "session",
message: {
type: "send_agent_audio",
agentId,
audio: base64Audio,
format,
isLast: true,
},
};
ws.send(msg);
console.log("[Session] Sent audio to agent:", agentId, format, audioBlob.size, "bytes");
} catch (error) {
console.error("[Session] Failed to send audio:", error);
throw error;
}
}, [ws]);
const createAgent = useCallback((options: { cwd: string; initialMode?: string; requestId?: string }) => {
const msg: WSInboundMessage = {
type: "session",
@@ -584,6 +619,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
pendingPermissions,
setPendingPermissions,
sendAgentMessage,
sendAgentAudio,
createAgent,
setAgentMode,
respondToPermission,
packages/app/src/hooks/use-audio-player.ts
+6 -16
View File
@@ -1,4 +1,4 @@
import { useState, useRef, useEffect } from 'react';
import { useState, useRef } from 'react';
import { initialize, playPCMData } from '@speechmatics/expo-two-way-audio';
interface QueuedAudio {
@@ -45,20 +45,6 @@ export function useAudioPlayer() {
const queueRef = useRef<QueuedAudio[]>([]);
const isProcessingQueueRef = useRef(false);
// Initialize Speechmatics audio on mount
useEffect(() => {
const initializeAudio = async () => {
try {
await initialize();
setAudioInitialized(true);
console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)');
} catch (error) {
console.error('[AudioPlayer] Failed to initialize audio:', error);
}
};
initializeAudio();
}, []);
async function play(audioData: Blob): Promise<number> {
return new Promise((resolve, reject) => {
// Add to queue with its promise handlers
@@ -112,8 +98,12 @@ export function useAudioPlayer() {
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
);
// Initialize audio if not already initialized
if (!audioInitialized) {
throw new Error('Audio not initialized');
console.log('[AudioPlayer] Initializing audio...');
await initialize();
setAudioInitialized(true);
console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)');
}
// Get PCM data from blob (server now sends PCM format)
packages/app/src/hooks/use-speechmatics-audio.ts
+18 -34
View File
@@ -149,26 +149,6 @@ export function useSpeechmaticsAudio(
const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration;
const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod;
// Initialize audio on mount
useEffect(() => {
const initializeAudio = async () => {
try {
await initialize();
setAudioInitialized(true);
console.log("[SpeechmaticsAudio] Audio initialized");
} catch (error) {
console.error("[SpeechmaticsAudio] Initialization error:", error);
const err = error instanceof Error ? error : new Error(String(error));
config.onError?.(err);
}
};
initializeAudio();
return () => {
tearDown();
};
}, []);
// Update segment duration timer
useEffect(() => {
if (!isDetecting && !isSpeaking) {
@@ -314,16 +294,20 @@ export function useSpeechmaticsAudio(
);
async function start(): Promise<void> {
if (!audioInitialized) {
throw new Error("Audio not initialized");
}
if (isActive) {
console.log("[SpeechmaticsAudio] Already active");
return;
}
try {
// Initialize audio if not already initialized
if (!audioInitialized) {
console.log("[SpeechmaticsAudio] Initializing audio...");
await initialize();
setAudioInitialized(true);
console.log("[SpeechmaticsAudio] Audio initialized");
}
console.log("[SpeechmaticsAudio] Starting audio capture...");
// Start recording
@@ -344,7 +328,16 @@ export function useSpeechmaticsAudio(
console.log("[SpeechmaticsAudio] Stopping audio capture...");
// Stop recording
toggleRecording(false);
if (isActive) {
toggleRecording(false);
}
// Tear down audio session
if (audioInitialized) {
tearDown();
setAudioInitialized(false);
console.log("[SpeechmaticsAudio] Audio torn down");
}
// Reset state
audioBufferRef.current = [];
@@ -383,15 +376,6 @@ export function useSpeechmaticsAudio(
});
}
// Cleanup on unmount
useEffect(() => {
return () => {
if (isActive) {
stop();
}
};
}, [isActive]);
return {
start,
stop,
packages/server/src/server/acp/agent-manager.ts
+15
View File
@@ -268,6 +268,20 @@ export class AgentManager {
throw new Error(`Agent ${agentId} is ${agent.status}`);
}
// Auto-cancel if agent is currently processing
// This allows users to interrupt stuck agents by sending a new message
if (agent.status === "processing") {
console.log(`[Agent ${agentId}] Auto-cancelling current task before new prompt`);
try {
await this.cancelAgent(agentId);
// Small delay to allow cancellation to propagate
await new Promise(resolve => setTimeout(resolve, 100));
} catch (error) {
console.warn(`[Agent ${agentId}] Cancel failed, continuing with new prompt:`, error);
// Continue anyway - the new prompt might still work
}
}
// Reset message IDs for new turn - ensures each prompt gets fresh responses
agent.currentAssistantMessageId = null;
agent.currentThoughtId = null;
@@ -784,6 +798,7 @@ export class AgentManager {
timestamp: new Date(),
notification: {
type: 'permission',
requestId, // Pass the requestId that we stored in pendingPermissions
request: params,
},
};
packages/server/src/server/acp/types.ts
+1 -1
View File
@@ -23,7 +23,7 @@ export interface EnrichedSessionNotification extends Omit<SessionNotification, '
*/
export type AgentNotification =
| { type: 'session'; notification: EnrichedSessionNotification }
| { type: 'permission'; request: RequestPermissionRequest }
| { type: 'permission'; requestId: string; request: RequestPermissionRequest }
| { type: 'status'; status: AgentStatus; error?: string };
/**
packages/server/src/server/session.ts
+2 -2
View File
@@ -178,14 +178,14 @@ export class Session {
type: "agent_permission_request",
payload: {
agentId,
requestId: uuidv4(), // Generate request ID
requestId: notification.requestId, // Use the requestId from AgentManager
sessionId: permissionRequest.sessionId,
toolCall: permissionRequest.toolCall,
options: permissionRequest.options,
},
});
console.log(
`[Session ${this.clientId}] Forwarded permission request for agent ${agentId}`
`[Session ${this.clientId}] Forwarded permission request for agent ${agentId} with requestId ${notification.requestId}`
);
return;
}