refactor: optimize audio initialization and fix agent interactions

- Lazy initialize audio (on-demand instead of mount) in both player and Speechmatics hooks
- Add audio session teardown on stop for proper cleanup
- Implement agent audio recording with sendAgentAudio in AgentInputArea
- Fix Android navigation race condition in CreateAgentModal (navigate before closing)
- Add animations to global footer and agent screen (FadeIn/FadeOut)
- Auto-cancel stuck agents when sending new messages
- Fix permission request ID propagation from AgentManager to Session
- Apply theme background color via AppContainer wrapper

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Mohamed Boudra
2025-10-23 13:15:18 +02:00
co-authored by Claude
parent 10189514dd
commit 74fffcd8a3
11 changed files with 139 additions and 76 deletions
@@ -120,6 +120,7 @@ interface SessionContextValue {
// Helpers
sendAgentMessage: (agentId: string, message: string) => void;
sendAgentAudio: (agentId: string, audioBlob: Blob) => Promise<void>;
createAgent: (options: { cwd: string; initialMode?: string; requestId?: string }) => void;
setAgentMode: (agentId: string, modeId: string) => void;
respondToPermission: (requestId: string, agentId: string, sessionId: string, selectedOptionIds: string[]) => void;
@@ -516,6 +517,40 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
ws.send(msg);
}, [ws]);
const sendAgentAudio = useCallback(async (agentId: string, audioBlob: Blob) => {
try {
// Convert blob to base64
const arrayBuffer = await audioBlob.arrayBuffer();
const bytes = new Uint8Array(arrayBuffer);
let binary = '';
for (let i = 0; i < bytes.length; i++) {
binary += String.fromCharCode(bytes[i]);
}
const base64Audio = btoa(binary);
// Determine format from MIME type
const format = audioBlob.type.split('/')[1] || 'm4a';
// Send audio message
const msg: WSInboundMessage = {
type: "session",
message: {
type: "send_agent_audio",
agentId,
audio: base64Audio,
format,
isLast: true,
},
};
ws.send(msg);
console.log("[Session] Sent audio to agent:", agentId, format, audioBlob.size, "bytes");
} catch (error) {
console.error("[Session] Failed to send audio:", error);
throw error;
}
}, [ws]);
const createAgent = useCallback((options: { cwd: string; initialMode?: string; requestId?: string }) => {
const msg: WSInboundMessage = {
type: "session",
@@ -584,6 +619,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
pendingPermissions,
setPendingPermissions,
sendAgentMessage,
sendAgentAudio,
createAgent,
setAgentMode,
respondToPermission,