diff --git a/packages/app/src/app/agent/[id].tsx b/packages/app/src/app/agent/[id].tsx
index 1cd3a6347..c45e39b84 100644
--- a/packages/app/src/app/agent/[id].tsx
+++ b/packages/app/src/app/agent/[id].tsx
@@ -1,5 +1,5 @@
-import { useEffect, useMemo } from "react";
-import { View, Text } from "react-native";
+import { useEffect, useMemo, useState } from "react";
+import { View, Text, ActivityIndicator } from "react-native";
import { useLocalSearchParams } from "expo-router";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
@@ -18,6 +18,7 @@ export default function AgentScreen() {
const { id } = useLocalSearchParams<{ id: string }>();
const { agents, agentStreamState, pendingPermissions, respondToPermission } = useSession();
const { registerFooterControls, unregisterFooterControls } = useFooterControls();
+ const [isContentReady, setIsContentReady] = useState(false);
// Keyboard animation
const { height: keyboardHeight } = useReanimatedKeyboardAnimation();
@@ -42,8 +43,39 @@ export default function AgentScreen() {
return ;
}, [id]);
+ // Defer heavy rendering to next frame for faster initial paint
useEffect(() => {
- if (!agentControls || !agent) {
+ let isMounted = true;
+ let frameId: number | null = null;
+ let timeoutId: ReturnType | null = null;
+
+ const markReady = () => {
+ if (isMounted) {
+ setIsContentReady(true);
+ }
+ };
+
+ if (typeof requestAnimationFrame === "function") {
+ frameId = requestAnimationFrame(markReady);
+ } else {
+ timeoutId = setTimeout(markReady, 0);
+ }
+
+ return () => {
+ isMounted = false;
+
+ if (frameId !== null && typeof cancelAnimationFrame === "function") {
+ cancelAnimationFrame(frameId);
+ }
+
+ if (timeoutId !== null) {
+ clearTimeout(timeoutId);
+ }
+ };
+ }, []);
+
+ useEffect(() => {
+ if (!agentControls || !agent || !isContentReady) {
unregisterFooterControls();
return;
}
@@ -53,7 +85,7 @@ export default function AgentScreen() {
return () => {
unregisterFooterControls();
};
- }, [agentControls, agent, registerFooterControls, unregisterFooterControls]);
+ }, [agentControls, agent, isContentReady, registerFooterControls, unregisterFooterControls]);
if (!agent) {
return (
@@ -73,20 +105,28 @@ export default function AgentScreen() {
{/* Content Area with Keyboard Animation */}
-
- respondToPermission(requestId, id!, agent.sessionId || "", [optionId])
- }
- />
+ {isContentReady ? (
+ <>
+
+ respondToPermission(requestId, id!, agent.sessionId || "", [optionId])
+ }
+ />
- {/* Footer area - status bar pinned within screen */}
-
-
-
+ {/* Footer area - status bar pinned within screen */}
+
+
+
+ >
+ ) : (
+
+
+
+ )}
);
@@ -103,6 +143,11 @@ const styles = StyleSheet.create((theme) => ({
footerContainer: {
backgroundColor: theme.colors.background,
},
+ loadingContainer: {
+ flex: 1,
+ alignItems: "center",
+ justifyContent: "center",
+ },
errorContainer: {
flex: 1,
alignItems: "center",
diff --git a/packages/app/src/components/agent-list.tsx b/packages/app/src/components/agent-list.tsx
index 5eaaef888..ff95dd669 100644
--- a/packages/app/src/components/agent-list.tsx
+++ b/packages/app/src/components/agent-list.tsx
@@ -63,7 +63,10 @@ export function AgentList({ agents }: AgentListProps) {
return (
[
+ styles.agentItem,
+ pressed && styles.agentItemPressed,
+ ]}
onPress={() => handleAgentPress(agent.id)}
>
@@ -112,6 +115,10 @@ const styles = StyleSheet.create((theme) => ({
marginBottom: theme.spacing[2],
backgroundColor: theme.colors.muted,
},
+ agentItemPressed: {
+ opacity: 0.7,
+ backgroundColor: theme.colors.accent,
+ },
agentContent: {
flex: 1,
},
diff --git a/packages/app/src/components/message.tsx b/packages/app/src/components/message.tsx
index dee45a5a0..24f63f74f 100644
--- a/packages/app/src/components/message.tsx
+++ b/packages/app/src/components/message.tsx
@@ -1,5 +1,5 @@
import { View, Text, Pressable, Animated } from "react-native";
-import { useState, useEffect, useRef } from "react";
+import { useState, useEffect, useRef, memo } from "react";
import Markdown from "react-native-markdown-display";
import {
Circle,
@@ -48,7 +48,7 @@ const userMessageStylesheet = StyleSheet.create((theme) => ({
},
}));
-export function UserMessage({ message, timestamp }: UserMessageProps) {
+export const UserMessage = memo(function UserMessage({ message, timestamp }: UserMessageProps) {
return (
@@ -56,7 +56,7 @@ export function UserMessage({ message, timestamp }: UserMessageProps) {
);
-}
+});
interface AssistantMessageProps {
message: string;
@@ -133,7 +133,7 @@ const assistantMessageStylesheet = StyleSheet.create((theme) => ({
},
}));
-export function AssistantMessage({
+export const AssistantMessage = memo(function AssistantMessage({
message,
timestamp,
isStreaming = false,
@@ -191,7 +191,7 @@ export function AssistantMessage({
)}
);
-}
+});
interface ActivityLogProps {
type: "system" | "info" | "success" | "error" | "artifact";
@@ -274,7 +274,7 @@ const activityLogStylesheet = StyleSheet.create((theme) => ({
},
}));
-export function ActivityLog({
+export const ActivityLog = memo(function ActivityLog({
type,
message,
timestamp,
@@ -374,7 +374,7 @@ export function ActivityLog({
);
-}
+});
interface ToolCallProps {
toolName: string;
@@ -486,7 +486,7 @@ const toolKindIcons: Record = {
// Add more mappings as needed
};
-export function ToolCall({
+export const ToolCall = memo(function ToolCall({
toolName,
kind,
args,
@@ -584,4 +584,4 @@ export function ToolCall({
);
-}
+});
diff --git a/packages/app/src/contexts/realtime-context.tsx b/packages/app/src/contexts/realtime-context.tsx
index c530d6880..c05f8ad84 100644
--- a/packages/app/src/contexts/realtime-context.tsx
+++ b/packages/app/src/contexts/realtime-context.tsx
@@ -1,4 +1,4 @@
-import { createContext, useContext, useState, ReactNode, useCallback } from "react";
+import { createContext, useContext, useState, ReactNode, useCallback, useEffect } from "react";
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
import { useSession } from "./session-context";
import { generateMessageId } from "@/types/stream";
@@ -31,7 +31,7 @@ interface RealtimeProviderProps {
}
export function RealtimeProvider({ children }: RealtimeProviderProps) {
- const { ws, audioPlayer, isPlayingAudio, setMessages } = useSession();
+ const { ws, audioPlayer, isPlayingAudio, setMessages, setVoiceDetectionFlags } = useSession();
const [isRealtimeMode, setIsRealtimeMode] = useState(false);
const realtimeAudio = useSpeechmaticsAudio({
@@ -82,6 +82,11 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
detectionGracePeriod: 200,
});
+ // Update voice detection flags whenever they change
+ useEffect(() => {
+ setVoiceDetectionFlags(realtimeAudio.isDetecting, realtimeAudio.isSpeaking);
+ }, [realtimeAudio.isDetecting, realtimeAudio.isSpeaking, setVoiceDetectionFlags]);
+
const startRealtime = useCallback(async () => {
try {
await realtimeAudio.start();
diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx
index c8e1941a5..aa14d3f6d 100644
--- a/packages/app/src/contexts/session-context.tsx
+++ b/packages/app/src/contexts/session-context.tsx
@@ -87,6 +87,9 @@ interface SessionContextValue {
audioPlayer: ReturnType;
isPlayingAudio: boolean;
setIsPlayingAudio: (playing: boolean) => void;
+
+ // Voice detection flags (updated by RealtimeContext)
+ setVoiceDetectionFlags: (isDetecting: boolean, isSpeaking: boolean) => void;
// Messages and stream state
messages: MessageEntry[];
@@ -133,7 +136,15 @@ interface SessionProviderProps {
export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
const ws = useWebSocket(serverUrl);
- const audioPlayer = useAudioPlayer();
+
+ // State for voice detection flags (will be set by RealtimeContext)
+ const isDetectingRef = useRef(false);
+ const isSpeakingRef = useRef(false);
+
+ const audioPlayer = useAudioPlayer({
+ isDetecting: () => isDetectingRef.current,
+ isSpeaking: () => isSpeakingRef.current,
+ });
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
const [messages, setMessages] = useState([]);
@@ -250,6 +261,20 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
}));
});
+ // Permission resolved - remove from pending
+ const unsubPermissionResolved = ws.on("agent_permission_resolved", (message) => {
+ if (message.type !== "agent_permission_resolved") return;
+ const { requestId, agentId, optionId } = message.payload;
+
+ console.log("[Session] Permission resolved:", requestId, "for agent:", agentId, "with option:", optionId);
+
+ setPendingPermissions((prev) => {
+ const next = new Map(prev);
+ next.delete(requestId);
+ return next;
+ });
+ });
+
// Audio output
const unsubAudioOutput = ws.on("audio_output", async (message) => {
if (message.type !== "audio_output") return;
@@ -458,6 +483,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
unsubAgentStatus();
unsubAgentUpdate();
unsubPermissionRequest();
+ unsubPermissionResolved();
unsubAudioOutput();
unsubActivity();
unsubChunk();
@@ -581,19 +607,21 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
};
ws.send(msg);
- // Remove from pending
- setPendingPermissions((prev) => {
- const next = new Map(prev);
- next.delete(requestId);
- return next;
- });
+ // Don't remove from pending here - wait for server confirmation via agent_permission_resolved
+ // This ensures UI stays in sync whether permission is accepted via UI or MCP
}, [ws]);
+ const setVoiceDetectionFlags = useCallback((isDetecting: boolean, isSpeaking: boolean) => {
+ isDetectingRef.current = isDetecting;
+ isSpeakingRef.current = isSpeaking;
+ }, []);
+
const value: SessionContextValue = {
ws,
audioPlayer,
isPlayingAudio,
setIsPlayingAudio,
+ setVoiceDetectionFlags,
messages,
setMessages,
currentAssistantMessage,
diff --git a/packages/app/src/hooks/use-audio-player.ts b/packages/app/src/hooks/use-audio-player.ts
index 5e7be1f98..5775cd604 100644
--- a/packages/app/src/hooks/use-audio-player.ts
+++ b/packages/app/src/hooks/use-audio-player.ts
@@ -42,18 +42,40 @@ function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array {
return pcm16k;
}
+export interface AudioPlayerOptions {
+ isDetecting?: () => boolean;
+ isSpeaking?: () => boolean;
+}
+
/**
* Hook for audio playback using Speechmatics two-way audio with echo cancellation
*/
-export function useAudioPlayer() {
+export function useAudioPlayer(options?: AudioPlayerOptions) {
const [isPlaying, setIsPlaying] = useState(false);
const [audioInitialized, setAudioInitialized] = useState(false);
const queueRef = useRef([]);
+ const suppressedQueueRef = useRef([]);
const isProcessingQueueRef = useRef(false);
const playbackTimeoutRef = useRef(null);
+ const checkIntervalRef = useRef(null);
async function play(audioData: Blob): Promise {
return new Promise((resolve, reject) => {
+ // Check if we should suppress playback due to voice detection/speaking
+ const shouldSuppress =
+ (options?.isDetecting && options.isDetecting()) ||
+ (options?.isSpeaking && options.isSpeaking());
+
+ if (shouldSuppress) {
+ console.log("[AudioPlayer] Suppressing playback - voice detection/speaking active");
+ // Add to suppressed queue instead
+ suppressedQueueRef.current.push({ audioData, resolve, reject });
+
+ // Start checking for when flags clear
+ startCheckingForClearFlags();
+ return;
+ }
+
// Add to queue with its promise handlers
queueRef.current.push({ audioData, resolve, reject });
@@ -64,6 +86,49 @@ export function useAudioPlayer() {
});
}
+ function startCheckingForClearFlags(): void {
+ // Already checking
+ if (checkIntervalRef.current !== null) {
+ return;
+ }
+
+ console.log("[AudioPlayer] Starting to check for clear flags");
+
+ checkIntervalRef.current = setInterval(() => {
+ const isStillBlocked =
+ (options?.isDetecting && options.isDetecting()) ||
+ (options?.isSpeaking && options.isSpeaking());
+
+ if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
+ console.log("[AudioPlayer] Flags cleared - moving suppressed queue to main queue");
+
+ // Move all suppressed items to main queue
+ const suppressedItems = [...suppressedQueueRef.current];
+ suppressedQueueRef.current = [];
+
+ // Add to front of main queue (they were waiting)
+ queueRef.current = [...suppressedItems, ...queueRef.current];
+
+ // Stop checking
+ if (checkIntervalRef.current !== null) {
+ clearInterval(checkIntervalRef.current);
+ checkIntervalRef.current = null;
+ }
+
+ // Start processing if not already
+ if (!isProcessingQueueRef.current) {
+ processQueue();
+ }
+ } else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
+ // No more suppressed items and flags are clear - stop checking
+ if (checkIntervalRef.current !== null) {
+ clearInterval(checkIntervalRef.current);
+ checkIntervalRef.current = null;
+ }
+ }
+ }, 100); // Check every 100ms
+ }
+
async function processQueue(): Promise {
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
return;
@@ -72,6 +137,20 @@ export function useAudioPlayer() {
isProcessingQueueRef.current = true;
while (queueRef.current.length > 0) {
+ // Before processing each item, check if flags became active
+ const shouldSuppress =
+ (options?.isDetecting && options.isDetecting()) ||
+ (options?.isSpeaking && options.isSpeaking());
+
+ if (shouldSuppress) {
+ console.log("[AudioPlayer] Flags became active during processing - moving remaining queue to suppressed");
+ // Move remaining queue to suppressed
+ suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
+ queueRef.current = [];
+ startCheckingForClearFlags();
+ break;
+ }
+
const item = queueRef.current.shift()!;
try {
const duration = await playAudio(item.audioData);
@@ -188,21 +267,45 @@ export function useAudioPlayer() {
setIsPlaying(false);
- // Reject all pending promises in the queue
+ // Reject all pending promises in the main queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error("Playback stopped"));
}
+ // Reject all pending promises in the suppressed queue
+ while (suppressedQueueRef.current.length > 0) {
+ const item = suppressedQueueRef.current.shift()!;
+ item.reject(new Error("Playback stopped"));
+ }
+
+ // Clear check interval
+ if (checkIntervalRef.current !== null) {
+ clearInterval(checkIntervalRef.current);
+ checkIntervalRef.current = null;
+ }
+
isProcessingQueueRef.current = false;
}
function clearQueue(): void {
- // Reject all pending promises in the queue
+ // Reject all pending promises in the main queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error("Queue cleared"));
}
+
+ // Reject all pending promises in the suppressed queue
+ while (suppressedQueueRef.current.length > 0) {
+ const item = suppressedQueueRef.current.shift()!;
+ item.reject(new Error("Queue cleared"));
+ }
+
+ // Clear check interval
+ if (checkIntervalRef.current !== null) {
+ clearInterval(checkIntervalRef.current);
+ checkIntervalRef.current = null;
+ }
}
return {