From b86e5fbb3fb1696cd07c08b6d337f9532a734908 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 22 Oct 2025 22:06:00 +0200 Subject: [PATCH] Based on the changes, I can see this is a significant feature addition that adds the ability to create agents and communicate with them through both text and audio, with a modal UI for agent creation and mode switching. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` feat: add agent creation modal and agent-specific messaging - Add CreateAgentModal component for creating new agents with mode selection - Implement agent-specific text and audio message routing - Add mode selector modal with visual mode switching UI - Introduce unified stream reducer pattern for agent activity - Refactor AgentStreamView to use StreamItem[] instead of raw notifications - Update ActiveProcesses to show orchestrator pill and support mode switching - Add message types: send_agent_message, send_agent_audio, create_agent_request, set_agent_mode - Implement agent audio transcription and message forwarding in session handler - Add mode badge UI showing current agent mode - Update tmux capture-pane to allowed commands 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude ``` --- packages/app/src/app/index.tsx | 549 +++++++++++++----- .../app/src/components/active-processes.tsx | 76 ++- .../app/src/components/agent-activity.tsx | 4 +- .../app/src/components/agent-stream-view.tsx | 298 ++++++---- .../app/src/components/create-agent-modal.tsx | 324 +++++++++++ packages/app/src/types/stream.ts | 319 ++++++++++ .../server/src/server/acp/agent-manager.ts | 14 +- packages/server/src/server/messages.ts | 45 +- packages/server/src/server/session.ts | 238 +++++++- 9 files changed, 1568 insertions(+), 299 deletions(-) create mode 100644 packages/app/src/components/create-agent-modal.tsx create mode 100644 packages/app/src/types/stream.ts diff --git a/packages/app/src/app/index.tsx b/packages/app/src/app/index.tsx index 60b08e429..98077942b 100644 --- a/packages/app/src/app/index.tsx +++ b/packages/app/src/app/index.tsx @@ -9,6 +9,7 @@ import { ScrollView, Animated, Keyboard, + Modal, } from "react-native"; import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; import ReanimatedAnimated, { @@ -47,6 +48,8 @@ import { ActiveProcesses } from "@/components/active-processes"; import { AgentStreamView } from "@/components/agent-stream-view"; import { ConversationSelector } from "@/components/conversation-selector"; import { VolumeMeter } from "@/components/volume-meter"; +import { reduceStreamUpdate, type StreamItem } from "@/types/stream"; +import { CreateAgentModal } from "@/components/create-agent-modal"; import { Settings, Mic, @@ -54,6 +57,8 @@ import { Square, AudioLines, MicOff, + Plus, + ChevronDown, } from "lucide-react-native"; import type { ActivityLogPayload, @@ -62,6 +67,7 @@ import type { } from "@server/server/messages"; import type { AgentStatus } from "@server/server/acp/types"; import type { SessionNotification } from "@agentclientprotocol/sdk"; +import { parseSessionUpdate } from "@/types/agent-activity"; type MessageEntry = | { @@ -277,12 +283,12 @@ export default function VoiceAssistantScreen() { onAudioSegment: (base64Audio: string) => { console.log("[App] Sending audio segment, length:", base64Audio.length); - // Send audio segment to server + // Send audio segment to server (realtime always goes to orchestrator) try { ws.send({ type: "session", message: { - type: "audio_chunk", + type: "realtime_audio_chunk", audio: base64Audio, format: "audio/wav", isLast: true, // Complete segment @@ -314,6 +320,9 @@ export default function VoiceAssistantScreen() { const [messages, setMessages] = useState([]); const [currentAssistantMessage, setCurrentAssistantMessage] = useState(""); const [isProcessingAudio, setIsProcessingAudio] = useState(false); + + // Agent stream state - unified chronological stream using reducer pattern + const [agentStreamState, setAgentStreamState] = useState>(new Map()); const [isPlayingAudio, setIsPlayingAudio] = useState(false); const [isRecording, setIsRecording] = useState(false); const [userInput, setUserInput] = useState(""); @@ -326,6 +335,12 @@ export default function VoiceAssistantScreen() { const [viewMode, setViewMode] = useState("orchestrator"); const [activeAgentId, setActiveAgentId] = useState(null); + // Agent creation modal state + const [showCreateAgentModal, setShowCreateAgentModal] = useState(false); + + // Mode selector modal state + const [showModeSelector, setShowModeSelector] = useState(false); + // Agent and command state const [agents, setAgents] = useState>(new Map()); const [commands, setCommands] = useState>(new Map()); @@ -389,8 +404,11 @@ export default function VoiceAssistantScreen() { "commands" ); - // Update agents - setAgents(new Map(agentsList.map((a) => [a.id, a as Agent]))); + // Update agents (convert createdAt string to Date) + setAgents(new Map(agentsList.map((a) => [a.id, { + ...a, + createdAt: new Date(a.createdAt), + } as Agent]))); // Update commands setCommands(new Map(commandsList.map((c) => [c.id, c as Command]))); @@ -402,7 +420,7 @@ export default function VoiceAssistantScreen() { const { agentId, status, type, currentModeId, availableModes } = message.payload; - console.log("[App] Agent created:", agentId); + console.log("[App] Agent created:", agentId, "currentModeId:", currentModeId, "availableModes:", availableModes); const agent: Agent = { id: agentId, @@ -415,6 +433,10 @@ export default function VoiceAssistantScreen() { setAgents((prev) => new Map(prev).set(agentId, agent)); setAgentUpdates((prev) => new Map(prev).set(agentId, [])); + + // Auto-switch to agent view + setActiveAgentId(agentId); + setViewMode("agent"); }); // Agent update handler @@ -424,6 +446,7 @@ export default function VoiceAssistantScreen() { console.log("[App] Agent update:", agentId); + // Store raw update for agent stream view setAgentUpdates((prev) => { const updated = new Map(prev); const updates = updated.get(agentId) || []; @@ -436,6 +459,15 @@ export default function VoiceAssistantScreen() { ]); return updated; }); + + // Reduce stream update using unified reducer pattern + setAgentStreamState((prev) => { + const currentStream = prev.get(agentId) || []; + const nextStream = reduceStreamUpdate(currentStream, notification, new Date(timestamp)); + const updated = new Map(prev); + updated.set(agentId, nextStream); + return updated; + }); }); // Agent status handler @@ -454,6 +486,8 @@ export default function VoiceAssistantScreen() { status: status as AgentStatus, sessionId: info.sessionId, error: info.error, + currentModeId: info.currentModeId, + availableModes: info.availableModes, }); } return updated; @@ -808,16 +842,44 @@ export default function VoiceAssistantScreen() { ) ); - // Send to server (properly typed) - const audioMessage: WSInboundMessage = { - type: "session", - message: { - type: "audio_chunk", - audio: base64Audio, - format: format, - isLast: true, - }, - }; + // Route audio based on view mode + let audioMessage: WSInboundMessage; + if (isRealtimeMode) { + // Send as realtime audio chunk to orchestrator (speech-to-speech with TTS) + audioMessage = { + type: "session", + message: { + type: "realtime_audio_chunk", + audio: base64Audio, + format: format, + isLast: true, + }, + }; + } else if (viewMode === "agent" && activeAgentId) { + // Send as agent audio (will be transcribed, no TTS) + audioMessage = { + type: "session", + message: { + type: "send_agent_audio", + agentId: activeAgentId, + audio: base64Audio, + format: format, + isLast: true, + }, + }; + } else { + // Send as regular audio to orchestrator (will be transcribed, no TTS) + audioMessage = { + type: "session", + message: { + type: "realtime_audio_chunk", + audio: base64Audio, + format: format, + isLast: true, + }, + }; + } + ws.send(audioMessage); console.log( @@ -903,6 +965,43 @@ export default function VoiceAssistantScreen() { // TODO: Implement cancel agent API call } + // Agent creation handler + function handleCreateAgent(workingDir: string, mode: string) { + console.log("[App] Creating agent in:", workingDir, "with mode:", mode); + + // Send create agent request to server + const message: WSInboundMessage = { + type: "session", + message: { + type: "create_agent_request", + cwd: workingDir, + initialMode: mode, + }, + }; + ws.send(message); + + // Close modal + setShowCreateAgentModal(false); + + // The agent_created event handler will switch to agent view automatically + } + + // Mode change handler + function handleModeChange(modeId: string) { + if (!activeAgentId) return; + + const message: WSInboundMessage = { + type: "session", + message: { + type: "set_agent_mode", + agentId: activeAgentId, + modeId, + }, + }; + ws.send(message); + setShowModeSelector(false); + } + // Text message handlers function handleSendMessage() { if (!userInput.trim() || !ws.isConnected) return; @@ -911,8 +1010,44 @@ export default function VoiceAssistantScreen() { audioPlayer.stop(); setIsPlayingAudio(false); - // Send message to server using the hook's method - ws.sendUserMessage(userInput); + // Route message based on view mode + if (isRealtimeMode) { + // Realtime mode always routes to orchestrator (handled by realtime audio) + ws.sendUserMessage(userInput); + } else if (viewMode === "agent" && activeAgentId) { + // Optimistically add user message to stream + setAgentStreamState((prev) => { + const currentStream = prev.get(activeAgentId) || []; + const nextStream = reduceStreamUpdate( + currentStream, + { + type: "sessionUpdate", + update: { + sessionUpdate: "user_message_chunk", + content: { type: "text", text: userInput }, + }, + }, + new Date() + ); + const updated = new Map(prev); + updated.set(activeAgentId, nextStream); + return updated; + }); + + // Send to agent + const message: WSInboundMessage = { + type: "session", + message: { + type: "send_agent_message", + agentId: activeAgentId, + text: userInput, + }, + }; + ws.send(message); + } else { + // Send to orchestrator + ws.sendUserMessage(userInput); + } // Clear input and reset streaming state setUserInput(""); @@ -1022,7 +1157,9 @@ export default function VoiceAssistantScreen() { function handleSelectConversation(newConversationId: string | null) { // Clear all state setMessages([]); + setAgentMessages(new Map()); setCurrentAssistantMessage(""); + setCurrentAgentMessages(new Map()); setUserInput(""); setArtifacts(new Map()); setCurrentArtifact(null); @@ -1045,32 +1182,11 @@ export default function VoiceAssistantScreen() { setConversationId(newConversationId); } - // Render agent stream view - if (viewMode === "agent" && activeAgentId) { - const agent = agents.get(activeAgentId); - const updates = agentUpdates.get(activeAgentId) || []; + // Calculate agent data (used when viewMode === "agent") + const agent = activeAgentId ? agents.get(activeAgentId) : null; + const streamItems = activeAgentId ? (agentStreamState.get(activeAgentId) || []) : []; - if (!agent) { - return ( - - Agent not found - - ); - } - - return ( - - ); - } - - // Render orchestrator view (main chat) + // Render main view with shared structure return ( {/* Fixed Header */} @@ -1086,6 +1202,16 @@ export default function VoiceAssistantScreen() { onSelectConversation={handleSelectConversation} websocket={ws} /> + setShowCreateAgentModal(true)} + disabled={!ws.isConnected} + style={[ + styles.settingsButton, + !ws.isConnected && styles.buttonDisabled, + ]} + > + + router.push("/settings")} style={styles.settingsButton} @@ -1100,10 +1226,10 @@ export default function VoiceAssistantScreen() { @@ -1111,100 +1237,115 @@ export default function VoiceAssistantScreen() { - {/* Scrollable Messages Area */} - - {messages.length === 0 && !currentAssistantMessage && ( - - OnTheGo - - What would you like to work on? - - - )} + {/* Conditionally render content based on view mode */} + {viewMode === "agent" && activeAgentId && agent ? ( + // Agent view - render AgentStreamView + + ) : viewMode === "agent" && activeAgentId && !agent ? ( + // Agent not found + + Agent not found + + ) : ( + // Orchestrator view - render scrollable messages + + {messages.length === 0 && !currentAssistantMessage && ( + + OnTheGo + + What would you like to work on? + + + )} - {messages.map((msg) => { - if (msg.type === "user") { - return ( - - ); - } + {messages.map((msg) => { + if (msg.type === "user") { + return ( + + ); + } - if (msg.type === "assistant") { - return ( - - ); - } + if (msg.type === "assistant") { + return ( + + ); + } - if (msg.type === "activity") { - return ( - - ); - } + if (msg.type === "activity") { + return ( + + ); + } - if (msg.type === "artifact") { - return ( - - ); - } + if (msg.type === "artifact") { + return ( + + ); + } - if (msg.type === "tool_call") { - return ( - - ); - } + if (msg.type === "tool_call") { + return ( + + ); + } - return null; - })} + return null; + })} - {/* Streaming assistant message */} - {currentAssistantMessage && ( - - )} - + {/* Streaming assistant message */} + {currentAssistantMessage && ( + + )} + + )} {/* Fixed Footer */} - {/* Buttons */} - + {/* Mode badge and buttons row */} + + {/* Session mode badge - only show in agent view */} + {viewMode === "agent" && activeAgentId && agent && ( + setShowModeSelector(true)} + style={({ pressed }) => [ + styles.modeBadge, + pressed && styles.modeBadgePressed, + ]} + > + + {agent.availableModes?.find(m => m.id === agent.currentModeId)?.name || agent.currentModeId || 'default'} + + + + )} + + {/* Buttons */} + {userInput.trim().length > 0 ? ( // Send button when text is entered )} + )} @@ -1351,6 +1511,53 @@ export default function VoiceAssistantScreen() { artifact={currentArtifact} onClose={handleCloseArtifact} /> + + {/* Create agent modal */} + setShowCreateAgentModal(false)} + onCreateAgent={handleCreateAgent} + /> + + {/* Mode selector modal */} + setShowModeSelector(false)} + > + setShowModeSelector(false)} + > + + {agent?.availableModes?.map((mode) => { + const isActive = mode.id === agent.currentModeId; + return ( + handleModeChange(mode.id)} + style={[ + styles.modeItem, + isActive && styles.modeItemActive, + ]} + > + {mode.name} + {mode.description && ( + {mode.description} + )} + + ); + })} + + + ); } @@ -1362,12 +1569,13 @@ const styles = StyleSheet.create((theme) => ({ }, agentNotFoundContainer: { flex: 1, - backgroundColor: theme.colors.background, alignItems: "center", justifyContent: "center", + padding: theme.spacing[6], }, agentNotFoundText: { - color: theme.colors.foreground, + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.lg, }, header: { backgroundColor: theme.colors.background, @@ -1446,15 +1654,20 @@ const styles = StyleSheet.create((theme) => ({ marginBottom: theme.spacing[3], maxHeight: 128, }, - buttonRow: { + controlsRow: { flexDirection: "row", alignItems: "center", - justifyContent: "flex-end", gap: theme.spacing[2], marginBottom: theme.spacing[1], paddingHorizontal: theme.spacing[4], paddingBottom: theme.spacing[2], }, + buttonRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + marginLeft: "auto", + }, realtimeButton: { width: 40, height: 40, @@ -1529,4 +1742,64 @@ const styles = StyleSheet.create((theme) => ({ fontSize: theme.fontSize.sm, fontFamily: "monospace", }, + modeBadge: { + flexDirection: 'row', + alignItems: 'center', + gap: theme.spacing[2], + paddingHorizontal: theme.spacing[3], + paddingVertical: theme.spacing[2], + backgroundColor: theme.colors.palette.blue[950], + borderRadius: theme.borderRadius.full, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.palette.blue[800], + }, + modeBadgePressed: { + backgroundColor: theme.colors.palette.blue[900], + borderColor: theme.colors.palette.blue[700], + }, + modeBadgeText: { + color: theme.colors.palette.blue[400], + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + textTransform: 'capitalize', + }, + modalOverlay: { + flex: 1, + backgroundColor: 'rgba(0,0,0,0.5)', + justifyContent: 'center', + alignItems: 'center', + }, + modeSelectorContent: { + backgroundColor: theme.colors.card, + borderRadius: theme.borderRadius.lg, + padding: theme.spacing[4], + minWidth: 280, + maxWidth: 320, + }, + modeItem: { + padding: theme.spacing[4], + borderRadius: theme.borderRadius.md, + marginBottom: theme.spacing[2], + backgroundColor: theme.colors.muted, + }, + modeItemActive: { + backgroundColor: theme.colors.primary, + }, + modeName: { + color: theme.colors.foreground, + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + marginBottom: theme.spacing[1], + }, + modeNameActive: { + color: theme.colors.primaryForeground, + }, + modeDescription: { + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.sm, + }, + modeDescriptionActive: { + color: theme.colors.primaryForeground, + opacity: theme.opacity[80], + }, })); diff --git a/packages/app/src/components/active-processes.tsx b/packages/app/src/components/active-processes.tsx index 3a16d5e6e..0412a0c3b 100644 --- a/packages/app/src/components/active-processes.tsx +++ b/packages/app/src/components/active-processes.tsx @@ -18,10 +18,10 @@ export interface ActiveProcessesProps { isDead: boolean; exitCode: number | null; }>; - activeProcessId: string | null; - activeProcessType: 'agent' | null; + viewMode: 'orchestrator' | 'agent'; + activeAgentId: string | null; onSelectAgent: (id: string) => void; - onBackToOrchestrator: () => void; + onSelectOrchestrator: () => void; } function getAgentStatusColor(status: AgentStatus): string { @@ -122,6 +122,9 @@ const styles = StyleSheet.create((theme) => ({ fontSize: theme.fontSize.xs, fontWeight: '500', }, + processTextActive: { + color: theme.colors.primaryForeground, + }, statusDot: { width: 8, height: 8, @@ -138,39 +141,43 @@ const styles = StyleSheet.create((theme) => ({ export function ActiveProcesses({ agents, commands, - activeProcessId, - activeProcessType, + viewMode, + activeAgentId, onSelectAgent, - onBackToOrchestrator, + onSelectOrchestrator, }: ActiveProcessesProps) { - const hasActiveProcess = activeProcessId !== null && activeProcessType !== null; - - if (agents.length === 0 && commands.length === 0) { + // Only show if there's at least one agent + if (agents.length === 0) { return null; } return ( - {hasActiveProcess && ( - - [styles.backButton, pressed && styles.backButtonActive]} - > - Back to Chat - - - )} - + {/* Orchestrator pill */} + [ + styles.processItem, + viewMode === 'orchestrator' ? styles.processItemActive : styles.processItemInactive, + pressed && { opacity: 0.7 }, + ]} + > + + Orchestrator + + + {/* Agent pills */} {agents.map((agent) => { - const modeName = getModeName(agent.currentModeId, agent.availableModes); - const isActive = activeProcessType === 'agent' && activeProcessId === agent.id; + const isActive = viewMode === 'agent' && activeAgentId === agent.id; return ( - {agent.id.substring(0, 8)} + {agent.id.substring(0, 8)} @@ -194,26 +204,6 @@ export function ActiveProcesses({ ); })} - - {commands.map((command) => { - const statusColor = command.isDead - ? command.exitCode === 0 - ? '#22c55e' - : '#ef4444' - : '#3b82f6'; - - return ( - - - - - {command.name || command.currentCommand.substring(0, 20)} - - - - - ); - })} ); diff --git a/packages/app/src/components/agent-activity.tsx b/packages/app/src/components/agent-activity.tsx index 8dc0f535d..bbf6d1f00 100644 --- a/packages/app/src/components/agent-activity.tsx +++ b/packages/app/src/components/agent-activity.tsx @@ -267,7 +267,7 @@ const stylesheet = StyleSheet.create((theme) => ({ marginBottom: theme.spacing[1], }, thoughtLabel: { - color: theme.colors.palette.purple[400], + color: theme.colors.palette.purple[500], fontSize: theme.fontSize.xs, fontWeight: theme.fontWeight.semibold, marginBottom: theme.spacing[1], @@ -319,7 +319,7 @@ const stylesheet = StyleSheet.create((theme) => ({ statusText: { color: theme.colors.foreground, fontSize: theme.fontSize.xs, - fontWeight: theme.fontWeight.medium, + fontWeight: theme.fontWeight.semibold, }, expandIcon: { color: theme.colors.mutedForeground, diff --git a/packages/app/src/components/agent-stream-view.tsx b/packages/app/src/components/agent-stream-view.tsx index 325c0d3df..5ac46cd68 100644 --- a/packages/app/src/components/agent-stream-view.tsx +++ b/packages/app/src/components/agent-stream-view.tsx @@ -1,11 +1,10 @@ -import { useEffect, useRef, useMemo } from 'react'; +import { useEffect, useRef } from 'react'; import { View, Text, ScrollView, Pressable } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { StyleSheet } from 'react-native-unistyles'; import type { AgentStatus } from '@server/server/acp/types'; -import type { SessionNotification } from '@agentclientprotocol/sdk'; -import { AgentActivityItem } from './agent-activity'; -import { parseSessionUpdate, groupActivities, type AgentActivity } from '@/types/agent-activity'; +import { AssistantMessage } from './message'; +import type { StreamItem } from '@/types/stream'; export interface AgentStreamViewProps { agentId: string; @@ -15,13 +14,7 @@ export interface AgentStreamViewProps { createdAt: Date; type: 'claude'; }; - updates: Array<{ - timestamp: Date; - notification: SessionNotification; - }>; - onBack: () => void; - onKillAgent: (agentId: string) => void; - onCancelAgent: (agentId: string) => void; + streamItems: StreamItem[]; } function getStatusColor(status: AgentStatus): string { @@ -60,6 +53,11 @@ function getStatusIcon(status: AgentStatus): string { } function formatTimestamp(date: Date): string { + // Handle invalid dates gracefully + if (!date || !(date instanceof Date) || isNaN(date.getTime())) { + return '--:--:--'; + } + return new Intl.DateTimeFormat('en-US', { hour: 'numeric', minute: '2-digit', @@ -68,124 +66,138 @@ function formatTimestamp(date: Date): string { }).format(date); } +function UserMessageCard({ text, timestamp }: { text: string; timestamp: Date }) { + return ( + + + {formatTimestamp(timestamp)} + + You + {text} + + ); +} + +function ThoughtCard({ text, timestamp }: { text: string; timestamp: Date }) { + return ( + + + {formatTimestamp(timestamp)} + + Thinking + {text} + + ); +} + +function ToolCallCard({ + title, + status, + toolKind, + timestamp +}: { + title: string; + status: 'pending' | 'in_progress' | 'completed' | 'failed'; + toolKind?: string; + timestamp: Date; +}) { + return ( + + + {formatTimestamp(timestamp)} + + + {toolKind || 'Tool'} + + {status} + + + {title} + + ); +} + export function AgentStreamView({ agentId, agent, - updates, - onBack, - onKillAgent, - onCancelAgent, + streamItems, }: AgentStreamViewProps) { const scrollViewRef = useRef(null); const insets = useSafeAreaInsets(); - // Parse and group activities - const groupedActivities = useMemo(() => { - // Parse all notifications into typed activities - const activities: AgentActivity[] = updates - .map((update) => { - const parsedUpdate = parseSessionUpdate(update.notification); - if (!parsedUpdate) return null; - - return { - timestamp: update.timestamp, - update: parsedUpdate, - }; - }) - .filter((activity): activity is AgentActivity => activity !== null); - - // Group consecutive text chunks and merge tool calls - return groupActivities(activities); - }, [updates]); - - // Auto-scroll to bottom when new updates arrive + // Auto-scroll to bottom when new items arrive useEffect(() => { scrollViewRef.current?.scrollToEnd({ animated: true }); - }, [groupedActivities]); - - const canCancel = agent.status === 'processing'; - const canKill = agent.status !== 'killed' && agent.status !== 'completed'; + }, [streamItems]); return ( - {/* Header */} - - {/* Back button */} - - - ← Back to Chat - - - - {/* Agent info */} - - - Agent: - - {agentId.substring(0, 8)} - - - - {getStatusIcon(agent.status)} - {agent.status} - - - - - Created: {formatTimestamp(agent.createdAt)} - - - {/* Control buttons */} - {(canCancel || canKill) && ( - - {canCancel && ( - onCancelAgent(agentId)} - style={stylesheet.cancelButton} - > - - Cancel - - - )} - {canKill && ( - onKillAgent(agentId)} - style={stylesheet.killButton} - > - - Kill - - - )} - - )} - - - {/* Updates list */} + {/* Content list */} - {groupedActivities.length === 0 ? ( + {streamItems.length === 0 ? ( - No updates yet.{'\n'}Waiting for agent activity... + Start chatting with this agent... ) : ( - groupedActivities.map((item, idx) => ( - - )) + streamItems.map((item) => { + switch (item.kind) { + case 'user_message': + return ( + + ); + + case 'assistant_message': + return ( + + ); + + case 'thought': + return ( + + ); + + case 'tool_call': + return ( + + ); + + case 'plan': + // TODO: Render plan component + return null; + + case 'activity_log': + case 'artifact': + // These are orchestrator-only, skip for now + return null; + + default: + return null; + } + }) )} @@ -197,6 +209,78 @@ const stylesheet = StyleSheet.create((theme) => ({ flex: 1, backgroundColor: theme.colors.background, }, + userMessageCard: { + backgroundColor: theme.colors.card, + borderRadius: theme.borderRadius.lg, + padding: theme.spacing[3], + marginBottom: theme.spacing[2], + borderLeftWidth: 3, + borderLeftColor: theme.colors.primary, + }, + userMessageLabel: { + color: theme.colors.primary, + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + marginBottom: theme.spacing[1], + }, + userMessageText: { + color: theme.colors.foreground, + fontSize: theme.fontSize.sm, + lineHeight: 20, + }, + timestamp: { + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.xs, + marginBottom: theme.spacing[1], + }, + thoughtCard: { + backgroundColor: theme.colors.card, + borderRadius: theme.borderRadius.lg, + padding: theme.spacing[3], + marginBottom: theme.spacing[2], + borderLeftWidth: 3, + borderLeftColor: theme.colors.palette.purple[500], + }, + thoughtLabel: { + color: theme.colors.palette.purple[500], + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + marginBottom: theme.spacing[1], + }, + thoughtText: { + color: theme.colors.foreground, + fontSize: theme.fontSize.sm, + lineHeight: 20, + fontStyle: 'italic', + }, + toolCallCard: { + backgroundColor: theme.colors.card, + borderRadius: theme.borderRadius.lg, + padding: theme.spacing[3], + marginBottom: theme.spacing[2], + borderLeftWidth: 3, + borderLeftColor: theme.colors.palette.blue[500], + }, + toolCallHeader: { + flexDirection: 'row', + justifyContent: 'space-between', + alignItems: 'center', + marginBottom: theme.spacing[1], + }, + toolCallLabel: { + color: theme.colors.palette.blue[500], + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + }, + toolCallStatus: { + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.medium, + }, + toolCallTitle: { + color: theme.colors.foreground, + fontSize: theme.fontSize.sm, + lineHeight: 20, + }, header: { backgroundColor: theme.colors.card, borderBottomWidth: theme.borderWidth[1], diff --git a/packages/app/src/components/create-agent-modal.tsx b/packages/app/src/components/create-agent-modal.tsx new file mode 100644 index 000000000..d54e1e761 --- /dev/null +++ b/packages/app/src/components/create-agent-modal.tsx @@ -0,0 +1,324 @@ +import { useState } from "react"; +import { + Modal, + View, + Text, + Pressable, + TextInput, + ScrollView, + KeyboardAvoidingView, + Platform, +} from "react-native"; +import { X } from "lucide-react-native"; +import { StyleSheet } from "react-native-unistyles"; +import { theme as defaultTheme } from "@/styles/theme"; + +interface CreateAgentModalProps { + isVisible: boolean; + onClose: () => void; + onCreateAgent: (workingDir: string, mode: string) => void; +} + +const MODES = [ + { + value: "plan", + label: "Plan", + description: "Plan and design before implementing", + }, + { + value: "bypassPermissions", + label: "Bypass Permissions", + description: "Skip permission prompts for faster execution", + }, +] as const; + +export function CreateAgentModal({ + isVisible, + onClose, + onCreateAgent, +}: CreateAgentModalProps) { + const [workingDir, setWorkingDir] = useState(""); + const [selectedMode, setSelectedMode] = useState("plan"); + const [errorMessage, setErrorMessage] = useState(""); + + function handleCreate() { + if (!workingDir.trim()) { + setErrorMessage("Working directory is required"); + return; + } + + onCreateAgent(workingDir.trim(), selectedMode); + handleClose(); + } + + function handleClose() { + setWorkingDir(""); + setSelectedMode("plan"); + setErrorMessage(""); + onClose(); + } + + return ( + + + + + + {/* Header */} + + Create New Agent + + + + + + {/* Form */} + + {/* Working Directory Input */} + + Working Directory + { + setWorkingDir(text); + setErrorMessage(""); + }} + autoCapitalize="none" + autoCorrect={false} + /> + {errorMessage ? ( + {errorMessage} + ) : ( + + Absolute path to the project directory + + )} + + + {/* Mode Selector */} + + Mode + + {MODES.map((mode) => ( + setSelectedMode(mode.value)} + style={[ + styles.modeOption, + selectedMode === mode.value && + styles.modeOptionSelected, + ]} + > + + + {selectedMode === mode.value && ( + + )} + + + {mode.label} + + {mode.description} + + + + + ))} + + + + {/* Action Buttons */} + + + Create Agent + + + + Cancel + + + + + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + modalOverlay: { + flex: 1, + backgroundColor: "rgba(0,0,0,0.5)", + }, + modalBackdrop: { + flex: 1, + }, + modalContent: { + backgroundColor: theme.colors.card, + borderTopLeftRadius: theme.spacing[6], + borderTopRightRadius: theme.spacing[6], + height: "75%", + }, + modalInner: { + flex: 1, + }, + header: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + padding: theme.spacing[6], + borderBottomWidth: theme.borderWidth[1], + borderBottomColor: theme.colors.border, + }, + headerTitle: { + color: theme.colors.foreground, + fontSize: theme.fontSize.lg, + fontWeight: theme.fontWeight.semibold, + }, + scrollView: { + flex: 1, + }, + scrollContent: { + padding: theme.spacing[6], + paddingBottom: theme.spacing[4] * 5, + }, + formSection: { + marginBottom: theme.spacing[6], + }, + label: { + color: theme.colors.foreground, + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + marginBottom: theme.spacing[2], + }, + input: { + backgroundColor: theme.colors.background, + color: theme.colors.foreground, + padding: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + fontSize: theme.fontSize.base, + }, + helperText: { + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.sm, + marginTop: theme.spacing[2], + }, + errorText: { + color: theme.colors.destructive, + fontSize: theme.fontSize.sm, + marginTop: theme.spacing[2], + }, + modeContainer: { + gap: theme.spacing[3], + }, + modeOption: { + backgroundColor: theme.colors.background, + borderRadius: theme.borderRadius.lg, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + padding: theme.spacing[4], + }, + modeOptionSelected: { + borderColor: theme.colors.palette.blue[500], + backgroundColor: theme.colors.muted, + }, + modeOptionContent: { + flexDirection: "row", + alignItems: "center", + }, + radioOuter: { + width: 20, + height: 20, + borderRadius: theme.borderRadius.full, + borderWidth: theme.borderWidth[2], + marginRight: theme.spacing[3], + alignItems: "center", + justifyContent: "center", + }, + radioOuterSelected: { + borderColor: theme.colors.palette.blue[500], + }, + radioOuterUnselected: { + borderColor: theme.colors.border, + }, + radioInner: { + width: 12, + height: 12, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.palette.blue[500], + }, + modeTextContainer: { + flex: 1, + }, + modeLabel: { + color: theme.colors.foreground, + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + marginBottom: theme.spacing[1], + }, + modeDescription: { + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.sm, + }, + buttonContainer: { + gap: theme.spacing[3], + marginTop: theme.spacing[4], + }, + createButton: { + backgroundColor: theme.colors.palette.blue[500], + paddingVertical: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + alignItems: "center", + }, + createButtonDisabled: { + backgroundColor: theme.colors.palette.blue[900], + opacity: theme.opacity[50], + }, + createButtonText: { + color: theme.colors.palette.white, + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + }, + cancelButton: { + backgroundColor: "transparent", + paddingVertical: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + alignItems: "center", + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + }, + cancelButtonText: { + color: theme.colors.foreground, + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + }, +})); diff --git a/packages/app/src/types/stream.ts b/packages/app/src/types/stream.ts new file mode 100644 index 000000000..24b46ccbd --- /dev/null +++ b/packages/app/src/types/stream.ts @@ -0,0 +1,319 @@ +import type { SessionNotification } from '@agentclientprotocol/sdk'; + +// Simple ID generator that works in React Native +let idCounter = 0; +function generateId(): string { + return `stream_${Date.now()}_${idCounter++}`; +} + +/** + * Unified stream item types for both orchestrator and agent views + */ +export type StreamItem = + | UserMessageItem + | AssistantMessageItem + | ThoughtItem + | ToolCallItem + | PlanItem + | ActivityLogItem + | ArtifactItem; + +export interface UserMessageItem { + kind: 'user_message'; + id: string; + text: string; + timestamp: Date; +} + +export interface AssistantMessageItem { + kind: 'assistant_message'; + id: string; + text: string; + timestamp: Date; +} + +export interface ThoughtItem { + kind: 'thought'; + id: string; + text: string; + timestamp: Date; +} + +export interface ToolCallItem { + kind: 'tool_call'; + id: string; + toolCallId: string; + title: string; + status: 'pending' | 'in_progress' | 'completed' | 'failed'; + toolKind?: 'read' | 'edit' | 'delete' | 'move' | 'search' | 'execute' | 'think' | 'fetch' | 'switch_mode' | 'other'; + rawInput?: Record; + rawOutput?: Record; + content?: unknown[]; + locations?: unknown[]; + timestamp: Date; +} + +export interface PlanItem { + kind: 'plan'; + id: string; + entries: Array<{ + content: string; + status: 'pending' | 'in_progress' | 'completed'; + priority: 'high' | 'medium' | 'low'; + }>; + timestamp: Date; +} + +export interface ActivityLogItem { + kind: 'activity_log'; + id: string; + activityType: 'system' | 'transcript' | 'assistant' | 'tool_call' | 'tool_result' | 'error'; + message: string; + metadata?: Record; + timestamp: Date; +} + +export interface ArtifactItem { + kind: 'artifact'; + id: string; + artifactId: string; + artifactType: 'markdown' | 'diff' | 'image' | 'code'; + title: string; + timestamp: Date; +} + +/** + * Parsed notification types (internal representation before converting to StreamItem) + */ +type ParsedNotification = + | { kind: 'user_message_chunk'; text: string } + | { kind: 'agent_message_chunk'; text: string } + | { kind: 'agent_thought_chunk'; text: string } + | { kind: 'tool_call'; toolCallId: string; title: string; status?: string; toolKind?: string; rawInput?: any; rawOutput?: any; content?: any[]; locations?: any[] } + | { kind: 'tool_call_update'; toolCallId: string; title?: string | null; status?: string | null; toolKind?: string | null; rawInput?: any; rawOutput?: any; content?: any[] | null; locations?: any[] | null } + | { kind: 'plan'; entries: Array<{ content: string; status: string; priority: string }> } + | { kind: 'current_mode_update'; currentModeId: string } + | { kind: 'available_commands_update'; availableCommands: Array<{ name: string; description: string }> } + | null; + +/** + * Parse SessionNotification into typed ParsedNotification + */ +function parseNotification(notification: SessionNotification | any): ParsedNotification { + const update = (notification as any).update; + + if (!update || !update.sessionUpdate) { + return null; + } + + const kind = update.sessionUpdate; + + switch (kind) { + case 'user_message_chunk': + return { + kind: 'user_message_chunk', + text: update.content?.text || '', + }; + + case 'agent_message_chunk': + return { + kind: 'agent_message_chunk', + text: update.content?.text || '', + }; + + case 'agent_thought_chunk': + return { + kind: 'agent_thought_chunk', + text: update.content?.text || '', + }; + + case 'tool_call': + return { + kind: 'tool_call', + toolCallId: update.toolCallId, + title: update.title, + status: update.status, + toolKind: update.kind, + rawInput: update.rawInput, + rawOutput: update.rawOutput, + content: update.content, + locations: update.locations, + }; + + case 'tool_call_update': + return { + kind: 'tool_call_update', + toolCallId: update.toolCallId, + title: update.title, + status: update.status, + toolKind: update.kind, + rawInput: update.rawInput, + rawOutput: update.rawOutput, + content: update.content, + locations: update.locations, + }; + + case 'plan': + return { + kind: 'plan', + entries: update.entries, + }; + + case 'current_mode_update': + return { + kind: 'current_mode_update', + currentModeId: update.currentModeId, + }; + + case 'available_commands_update': + return { + kind: 'available_commands_update', + availableCommands: update.availableCommands, + }; + + default: + return null; + } +} + +/** + * Core reducer function that processes stream updates one at a time + * + * This function handles: + * - Message chunks (accumulates text into single message) + * - Tool call updates (finds and merges with existing) + * - New items (appends to stream) + * + * Can be used for: + * - Hydration: updates.reduce(reduceStreamUpdate, []) + * - Real-time: setState(prev => reduceStreamUpdate(prev, update)) + */ +export function reduceStreamUpdate( + state: StreamItem[], + notification: SessionNotification | any, + timestamp: Date +): StreamItem[] { + const parsed = parseNotification(notification); + + if (!parsed) { + return state; + } + + // User message chunks - always create new message (they're complete from server) + if (parsed.kind === 'user_message_chunk') { + return [...state, { + kind: 'user_message', + id: generateId(), + text: parsed.text, + timestamp, + }]; + } + + // Agent message chunks - accumulate into last assistant message or create new + if (parsed.kind === 'agent_message_chunk') { + const last = state[state.length - 1]; + if (last?.kind === 'assistant_message') { + // Append to existing message + return [ + ...state.slice(0, -1), + { + ...last, + text: last.text + parsed.text, + }, + ]; + } + // Create new message + return [...state, { + kind: 'assistant_message', + id: generateId(), + text: parsed.text, + timestamp, + }]; + } + + // Thought chunks - accumulate into last thought or create new + if (parsed.kind === 'agent_thought_chunk') { + const last = state[state.length - 1]; + if (last?.kind === 'thought') { + // Append to existing thought + return [ + ...state.slice(0, -1), + { + ...last, + text: last.text + parsed.text, + }, + ]; + } + // Create new thought + return [...state, { + kind: 'thought', + id: generateId(), + text: parsed.text, + timestamp, + }]; + } + + // Tool call updates - find existing and merge + if (parsed.kind === 'tool_call_update') { + return state.map(item => { + if (item.kind === 'tool_call' && item.toolCallId === parsed.toolCallId) { + return { + ...item, + ...(parsed.title !== undefined && parsed.title !== null ? { title: parsed.title } : {}), + ...(parsed.status !== undefined && parsed.status !== null ? { status: parsed.status as any } : {}), + ...(parsed.toolKind !== undefined && parsed.toolKind !== null ? { toolKind: parsed.toolKind as any } : {}), + ...(parsed.rawInput ? { rawInput: { ...item.rawInput, ...parsed.rawInput } } : {}), + ...(parsed.rawOutput ? { rawOutput: { ...item.rawOutput, ...parsed.rawOutput } } : {}), + ...(parsed.content !== undefined && parsed.content !== null ? { content: parsed.content } : {}), + ...(parsed.locations !== undefined && parsed.locations !== null ? { locations: parsed.locations } : {}), + timestamp, + }; + } + return item; + }); + } + + // New tool call + if (parsed.kind === 'tool_call') { + return [...state, { + kind: 'tool_call', + id: parsed.toolCallId, + toolCallId: parsed.toolCallId, + title: parsed.title, + status: (parsed.status as any) || 'pending', + toolKind: parsed.toolKind as any, + rawInput: parsed.rawInput, + rawOutput: parsed.rawOutput, + content: parsed.content, + locations: parsed.locations, + timestamp, + }]; + } + + // Plan - replace existing or create new + if (parsed.kind === 'plan') { + const withoutPlan = state.filter(item => item.kind !== 'plan'); + return [...withoutPlan, { + kind: 'plan', + id: generateId(), + entries: parsed.entries as any, + timestamp, + }]; + } + + // Mode updates and other types don't create stream items + // They're handled separately in state management + return state; +} + +/** + * Hydrate stream state from batch of notifications + */ +export function hydrateStreamState( + notifications: Array<{ timestamp: Date; notification: SessionNotification }> +): StreamItem[] { + return notifications.reduce( + (state, { notification, timestamp }) => reduceStreamUpdate(state, notification, timestamp), + [] as StreamItem[] + ); +} diff --git a/packages/server/src/server/acp/agent-manager.ts b/packages/server/src/server/acp/agent-manager.ts index 078e27a4e..294723b0c 100644 --- a/packages/server/src/server/acp/agent-manager.ts +++ b/packages/server/src/server/acp/agent-manager.ts @@ -180,13 +180,15 @@ export class AgentManager { }); // Create a new session - const sessionResponse = await agent.connection.newSession({ + const sessionParams = { cwd, mcpServers: [], - ...(options.initialMode ? { initialMode: options.initialMode } : {}), - }); + }; + console.log(`[Agent ${agentId}] newSession params:`, sessionParams); + const sessionResponse = await agent.connection.newSession(sessionParams); agent.sessionId = sessionResponse.sessionId; + console.log(`[Agent ${agentId}] newSession response:`, JSON.stringify(sessionResponse, null, 2)); // Store session modes from response if (sessionResponse.modes) { @@ -200,6 +202,12 @@ export class AgentManager { console.log(`[Agent ${agentId}] Session created:`, sessionResponse.sessionId); } + // Set initial mode if requested (must be done after session creation) + if (options.initialMode && agent.sessionId) { + console.log(`[Agent ${agentId}] Setting initial mode to: ${options.initialMode}`); + await this.setSessionMode(agentId, options.initialMode); + } + agent.status = "ready"; // If an initial prompt was provided, send it diff --git a/packages/server/src/server/messages.ts b/packages/server/src/server/messages.ts index 43b062a4b..7b85c305e 100644 --- a/packages/server/src/server/messages.ts +++ b/packages/server/src/server/messages.ts @@ -7,11 +7,10 @@ import { z } from "zod"; export const UserTextMessageSchema = z.object({ type: z.literal("user_text"), text: z.string(), - disableTTS: z.boolean().optional(), }); -export const AudioChunkMessageSchema = z.object({ - type: z.literal("audio_chunk"), +export const RealtimeAudioChunkMessageSchema = z.object({ + type: z.literal("realtime_audio_chunk"), audio: z.string(), // base64 encoded format: z.string(), isLast: z.boolean(), @@ -45,15 +44,45 @@ export const SetRealtimeModeMessageSchema = z.object({ enabled: z.boolean(), }); +export const SendAgentMessageSchema = z.object({ + type: z.literal("send_agent_message"), + agentId: z.string(), + text: z.string(), +}); + +export const SendAgentAudioSchema = z.object({ + type: z.literal("send_agent_audio"), + agentId: z.string(), + audio: z.string(), // base64 encoded + format: z.string(), + isLast: z.boolean(), +}); + +export const CreateAgentRequestMessageSchema = z.object({ + type: z.literal("create_agent_request"), + cwd: z.string(), + initialMode: z.string().optional(), +}); + +export const SetAgentModeMessageSchema = z.object({ + type: z.literal("set_agent_mode"), + agentId: z.string(), + modeId: z.string(), +}); + export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ UserTextMessageSchema, - AudioChunkMessageSchema, + RealtimeAudioChunkMessageSchema, AbortRequestMessageSchema, AudioPlayedMessageSchema, LoadConversationRequestMessageSchema, ListConversationsRequestMessageSchema, DeleteConversationRequestMessageSchema, SetRealtimeModeMessageSchema, + SendAgentMessageSchema, + SendAgentAudioSchema, + CreateAgentRequestMessageSchema, + SetAgentModeMessageSchema, ]); export type SessionInboundMessage = z.infer; @@ -274,6 +303,14 @@ export type DeleteConversationResponseMessage = z.infer; +// Type exports for inbound message types +export type UserTextMessage = z.infer; +export type RealtimeAudioChunkMessage = z.infer; +export type SendAgentMessage = z.infer; +export type SendAgentAudio = z.infer; +export type CreateAgentRequestMessage = z.infer; +export type SetAgentModeMessage = z.infer; + // ============================================================================ // WebSocket Level Messages (wraps session messages) // ============================================================================ diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index c7f0b5658..929d5739e 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -286,7 +286,7 @@ export class Session { await this.handleUserText(msg.text); break; - case "audio_chunk": + case "realtime_audio_chunk": await this.handleAudioChunk(msg); break; @@ -313,6 +313,22 @@ export class Session { case "set_realtime_mode": this.handleSetRealtimeMode(msg.enabled); break; + + case "send_agent_message": + await this.handleSendAgentMessage(msg.agentId, msg.text); + break; + + case "send_agent_audio": + await this.handleSendAgentAudio(msg); + break; + + case "create_agent_request": + await this.handleCreateAgentRequest(msg.cwd, msg.initialMode); + break; + + case "set_agent_mode": + await this.handleSetAgentMode(msg.agentId, msg.modeId); + break; } } catch (error: any) { console.error( @@ -423,6 +439,219 @@ export class Session { ); } + /** + * Handle text message to agent + */ + private async handleSendAgentMessage(agentId: string, text: string): Promise { + console.log( + `[Session ${this.clientId}] Sending text to agent ${agentId}: ${text.substring(0, 50)}...` + ); + + try { + // Emit user message notification before sending to agent + // (Claude Code ACP doesn't echo user messages, so we do it manually) + this.emit({ + type: "agent_update", + payload: { + agentId, + timestamp: new Date().toISOString(), + notification: { + type: "sessionUpdate", + update: { + sessionUpdate: "user_message_chunk", + content: { + type: "text", + text: text, + }, + }, + } as any, + }, + }); + + await this.agentManager.sendPrompt(agentId, text); + console.log(`[Session ${this.clientId}] Sent text to agent ${agentId}`); + } catch (error: any) { + console.error( + `[Session ${this.clientId}] Failed to send text to agent ${agentId}:`, + error + ); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "error", + content: `Failed to send message to agent: ${error.message}`, + }, + }); + throw error; + } + } + + /** + * Handle audio message to agent (transcribe then send) + */ + private async handleSendAgentAudio( + msg: Extract + ): Promise { + const { agentId, audio, format, isLast } = msg; + + // Decode base64 + const audioBuffer = Buffer.from(audio, "base64"); + + // For now, we'll process each audio segment immediately + // In the future, we might want to buffer chunks similar to realtime audio + if (!isLast) { + console.log( + `[Session ${this.clientId}] Buffering agent audio chunk for agent ${agentId}` + ); + // TODO: Implement buffering if needed + return; + } + + console.log( + `[Session ${this.clientId}] Transcribing audio for agent ${agentId}` + ); + + try { + // Transcribe the audio + const result = await this.sttManager.transcribe(audioBuffer, format); + + const transcriptText = result.text.trim(); + if (!transcriptText) { + console.log( + `[Session ${this.clientId}] Empty transcription for agent ${agentId}, ignoring` + ); + return; + } + + console.log( + `[Session ${this.clientId}] Transcribed audio for agent ${agentId}: ${transcriptText}` + ); + + // Send transcribed text to agent + await this.agentManager.sendPrompt(agentId, transcriptText); + console.log(`[Session ${this.clientId}] Sent transcribed text to agent ${agentId}`); + } catch (error: any) { + console.error( + `[Session ${this.clientId}] Failed to process audio for agent ${agentId}:`, + error + ); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "error", + content: `Failed to process audio for agent: ${error.message}`, + }, + }); + throw error; + } + } + + /** + * Handle create agent request + */ + private async handleCreateAgentRequest(cwd: string, initialMode?: string): Promise { + console.log( + `[Session ${this.clientId}] Creating agent in ${cwd} with mode ${initialMode || "default"}` + ); + + try { + const agentId = await this.agentManager.createAgent({ + cwd, + initialMode, + }); + + console.log(`[Session ${this.clientId}] Created agent ${agentId}`); + + // Get agent info + const agentInfo = this.agentManager.listAgents().find(a => a.id === agentId); + console.log(`[Session ${this.clientId}] Agent info:`, { + currentModeId: agentInfo?.currentModeId, + availableModes: agentInfo?.availableModes + }); + + // Subscribe to agent updates + this.subscribeToAgent(agentId); + + // Emit agent_created message + this.emit({ + type: "agent_created", + payload: { + agentId, + status: agentInfo?.status || "initializing", + type: "claude", + currentModeId: agentInfo?.currentModeId, + availableModes: agentInfo?.availableModes, + }, + }); + console.log(`[Session ${this.clientId}] Emitted agent_created with currentModeId:`, agentInfo?.currentModeId); + } catch (error: any) { + console.error( + `[Session ${this.clientId}] Failed to create agent:`, + error + ); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "error", + content: `Failed to create agent: ${error.message}`, + }, + }); + throw error; + } + } + + /** + * Handle set agent mode request + */ + private async handleSetAgentMode(agentId: string, modeId: string): Promise { + console.log(`[Session ${this.clientId}] Setting agent ${agentId} mode to ${modeId}`); + + try { + await this.agentManager.setSessionMode(agentId, modeId); + console.log(`[Session ${this.clientId}] Agent ${agentId} mode set to ${modeId}`); + + // Emit agent_status to notify client of mode change + const info = this.agentManager.listAgents().find(a => a.id === agentId); + if (info) { + this.emit({ + type: "agent_status", + payload: { + agentId, + status: info.status, + info: { + id: info.id, + status: info.status, + createdAt: info.createdAt, + type: info.type, + sessionId: info.sessionId ?? undefined, + error: info.error ?? undefined, + currentModeId: info.currentModeId ?? undefined, + availableModes: info.availableModes ?? undefined, + }, + }, + }); + } + } catch (error: any) { + console.error(`[Session ${this.clientId}] Failed to set agent mode:`, error); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "error", + content: `Failed to set agent mode: ${error.message}`, + }, + }); + throw error; + } + } + /** * Send current session state (live agents and commands) to client */ @@ -431,6 +660,11 @@ export class Session { // Get live agents with session modes const agents = this.agentManager?.listAgents() || []; + // Subscribe to all existing agents (in case of reconnection) + for (const agent of agents) { + this.subscribeToAgent(agent.id); + } + // Get live commands from terminal manager let commands: any[] = []; if (this.terminalManager) { @@ -503,7 +737,7 @@ export class Session { * Handle audio chunk for buffering and transcription */ private async handleAudioChunk( - msg: Extract + msg: Extract ): Promise { // Decode base64 const audioBuffer = Buffer.from(msg.audio, "base64");