import { useState, useEffect, useRef } from "react"; import { Mic, Send, Radio, X } from "lucide-react"; import { useWebSocket } from "./hooks/useWebSocket"; import { createAudioPlayer } from "./lib/audio-playback"; import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture"; import { createRealtimeVAD, float32ArrayToBlob, type RealtimeVAD, } from "./lib/audio-realtime"; import { ToolCallCard } from "./components/ToolCallCard"; import { ArtifactDrawer, type Artifact } from "./components/ArtifactDrawer"; import { ConversationSelector } from "./components/ConversationSelector"; import { ActiveProcesses } from "./components/ActiveProcesses"; import { AgentStreamView } from "./components/AgentStreamView"; import { getCurrentConversationId, setCurrentConversationId, clearCurrentConversationId, } from "./lib/conversation-storage"; import type { AgentStatus } from "../server/acp/types.js"; import type { SessionNotification } from "@agentclientprotocol/sdk"; import "./App.css"; type LogEntry = | { type: "system" | "info" | "success" | "error" | "user" | "assistant"; id: string; timestamp: number; message: string; metadata?: Record; } | { type: "tool_call"; id: string; timestamp: number; toolName: string; args: any; result?: any; error?: any; status: "executing" | "completed" | "failed"; } | { type: "artifact"; id: string; timestamp: number; artifactId: string; artifactType: string; title: string; }; function App() { const [logs, setLogs] = useState([ { id: "1", timestamp: Date.now(), type: "system", message: "System initialized", }, ]); const [userInput, setUserInput] = useState(""); const [currentAssistantMessage, setCurrentAssistantMessage] = useState(""); const [isProcessingAudio, setIsProcessingAudio] = useState(false); const [isPlayingAudio, setIsPlayingAudio] = useState(false); const [isRecording, setIsRecording] = useState(false); const [isRealtimeMode, setIsRealtimeMode] = useState(false); const [isSpeechDetected, setIsSpeechDetected] = useState(false); const [isVADLoading, setIsVADLoading] = useState(false); const [currentArtifact, setCurrentArtifact] = useState(null); const [artifacts, setArtifacts] = useState>(new Map()); const [selectedConversationId, setSelectedConversationId] = useState< string | null >(() => getCurrentConversationId()); const [agents, setAgents] = useState< Map< string, { id: string; status: AgentStatus; createdAt: Date; type: "claude" } > >(new Map()); const [agentUpdates, setAgentUpdates] = useState< Map> >(new Map()); const [activeView, setActiveView] = useState< "orchestrator" | "terminal" | "agent" >("orchestrator"); const [activeProcessId, setActiveProcessId] = useState(null); const logEndRef = useRef(null); const audioPlayerRef = useRef(createAudioPlayer()); const recorderRef = useRef(null); const vadRef = useRef(null); const textareaRef = useRef(null); // WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod const wsUrl = `${window.location.protocol === "https:" ? "wss" : "ws"}://${ window.location.host }/ws`; const ws = useWebSocket(wsUrl, selectedConversationId); useEffect(() => { recorderRef.current = createAudioRecorder(); // Cleanup VAD on unmount return () => { if (vadRef.current) { vadRef.current.destroy(); } }; }, []); useEffect(() => { if (textareaRef.current) { textareaRef.current.style.height = "auto"; const newHeight = Math.max(textareaRef.current.scrollHeight, 44); textareaRef.current.style.height = `${newHeight}px`; } }, [userInput]); const addLog = ( type: "system" | "info" | "success" | "error" | "user" | "assistant", message: string, metadata?: Record ) => { setLogs((prev) => [ ...prev, { id: Date.now().toString(), timestamp: Date.now(), type, message, metadata, }, ]); }; useEffect(() => { // Listen for status messages const unsubStatus = ws.on("status", (message) => { if (message.type !== "status") return; const msg = "message" in message.payload ? String(message.payload.message) : undefined; addLog("info", msg || `Status: ${message.payload.status}`); }); // Listen for activity log messages const unsubActivity = ws.on("activity_log", (message) => { if (message.type !== "activity_log") return; const data = message.payload; // Handle tool calls if (data.type === "tool_call" && data.metadata) { const { toolCallId, toolName, arguments: args, } = data.metadata as { toolCallId: string; toolName: string; arguments: unknown; }; setLogs((prev) => [ ...prev, { type: "tool_call", id: toolCallId, timestamp: Date.now(), toolName, args, status: "executing", }, ]); return; } if (data.type === "tool_result" && data.metadata) { const { toolCallId, result } = data.metadata as { toolCallId: string; result: unknown; }; setLogs((prev) => prev.map((log) => log.type === "tool_call" && log.id === toolCallId ? { ...log, result, status: "completed" as const } : log ) ); return; } // Handle tool errors - update tool call status to failed if ( data.type === "error" && data.metadata && "toolCallId" in data.metadata ) { const { toolCallId, error } = data.metadata as { toolCallId: string; error: unknown; }; setLogs((prev) => prev.map((log) => log.type === "tool_call" && log.id === toolCallId ? { ...log, error, status: "failed" as const } : log ) ); // Don't return - still add the error message to the log } // Map activity log types to UI log types let logType: | "system" | "info" | "success" | "error" | "user" | "assistant" = "info"; if (data.type === "transcript") logType = "user"; else if (data.type === "assistant") logType = "assistant"; else if (data.type === "error") logType = "error"; addLog(logType, data.content, data.metadata); // Clear streaming state when assistant segment is received if (data.type === "assistant") { setCurrentAssistantMessage(""); } }); // Listen for streaming assistant chunks const unsubChunk = ws.on("assistant_chunk", (message) => { if (message.type !== "assistant_chunk") return; setCurrentAssistantMessage((prev) => prev + message.payload.chunk); }); // Listen for transcription results const unsubTranscription = ws.on("transcription_result", (message) => { if (message.type !== "transcription_result") return; setIsProcessingAudio(false); const transcriptText = message.payload.text.trim(); if (!transcriptText) { // Empty transcription - false positive, resume playback console.log( "[App] Empty transcription (false positive) - resuming playback" ); audioPlayerRef.current.resume(); } else { // Has content - real speech detected // Server will handle abort, just stop playback on client console.log("[App] Transcription received - stopping playback"); audioPlayerRef.current.stop(); setIsPlayingAudio(false); setCurrentAssistantMessage(""); } }); // Listen for conversation loaded confirmation const unsubConversationLoaded = ws.on("conversation_loaded", (message) => { if (message.type !== "conversation_loaded") return; const { conversationId, messageCount } = message.payload; // Save to localStorage for persistence setCurrentConversationId(conversationId); addLog( "success", `Loaded conversation with ${messageCount} messages` ); }); // Listen for artifacts const unsubArtifact = ws.on("artifact", (message) => { if (message.type !== "artifact") return; const artifact = message.payload; // Add to artifacts map setArtifacts((prev) => { const newMap = new Map(prev); newMap.set(artifact.id, artifact); return newMap; }); // Add as clickable log entry setLogs((prev) => [ ...prev, { type: "artifact" as const, id: Date.now().toString(), timestamp: Date.now(), artifactId: artifact.id, artifactType: artifact.type, title: artifact.title, }, ]); // Show drawer by default setCurrentArtifact(artifact); }); // Listen for audio output (TTS) const unsubAudioOutput = ws.on("audio_output", async (message) => { if (message.type !== "audio_output") return; const data = message.payload; try { setIsPlayingAudio(true); // Decode base64 audio const binaryString = atob(data.audio); const bytes = new Uint8Array(binaryString.length); for (let i = 0; i < binaryString.length; i++) { bytes[i] = binaryString.charCodeAt(i); } // Create blob with correct mime type const mimeType = data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`; const audioBlob = new Blob([bytes], { type: mimeType }); // Play audio await audioPlayerRef.current.play(audioBlob); // Send confirmation back to server ws.send({ type: "session", message: { type: "audio_played", id: data.id, }, }); setIsPlayingAudio(false); } catch (error: any) { console.error("[App] Audio playback error:", error); addLog("error", `Audio playback failed: ${error.message}`); setIsPlayingAudio(false); } }); // Listen for agent created const unsubAgentCreated = ws.on("agent_created", (message) => { if (message.type !== "agent_created") return; const { agentId, status, type } = message.payload; setAgents((prev) => { const newMap = new Map(prev); newMap.set(agentId, { id: agentId, status: status as AgentStatus, createdAt: new Date(), type, }); return newMap; }); addLog("success", `Agent ${agentId.substring(0, 8)} created`); }); // Listen for agent updates const unsubAgentUpdate = ws.on("agent_update", (message) => { if (message.type !== "agent_update") return; const { agentId, timestamp, notification } = message.payload; setAgentUpdates((prev) => { const newMap = new Map(prev); const updates = newMap.get(agentId) || []; newMap.set(agentId, [ ...updates, { timestamp: new Date(timestamp), notification: notification as SessionNotification, }, ]); return newMap; }); }); // Listen for agent status changes const unsubAgentStatus = ws.on("agent_status", (message) => { if (message.type !== "agent_status") return; const { agentId, status } = message.payload; setAgents((prev) => { const newMap = new Map(prev); const agent = newMap.get(agentId); if (agent) { newMap.set(agentId, { ...agent, status: status as AgentStatus, }); } return newMap; }); addLog("info", `Agent ${agentId.substring(0, 8)}: ${status}`); }); return () => { unsubStatus(); unsubActivity(); unsubChunk(); unsubTranscription(); unsubConversationLoaded(); unsubArtifact(); unsubAudioOutput(); unsubAgentCreated(); unsubAgentUpdate(); unsubAgentStatus(); }; }, [ws]); useEffect(() => { if (ws.isConnected) { addLog("success", "WebSocket connected"); } else { addLog("system", "WebSocket disconnected"); } }, [ws.isConnected]); // Auto-scroll to bottom when new logs are added useEffect(() => { logEndRef.current?.scrollIntoView({ behavior: "smooth" }); }, [logs, currentAssistantMessage]); const handleSendMessage = () => { if (!userInput.trim() || !ws.isConnected) return; // Stop any currently playing audio and clear the queue audioPlayerRef.current.stop(); setIsPlayingAudio(false); // Send message to server ws.sendUserMessage(userInput); // Clear input and reset streaming state setUserInput(""); setCurrentAssistantMessage(""); }; const handleToggleRecording = async () => { const recorder = recorderRef.current; if (!recorder || !ws.isConnected) return; try { if (isRecording) { console.log("[App] Stopping recording..."); const audioBlob = await recorder.stop(); setIsRecording(false); const format = audioBlob.type || "audio/webm"; console.log( `[App] Recording complete: ${audioBlob.size} bytes, format: ${format}` ); setIsProcessingAudio(true); addLog("info", "Sending audio to server..."); const arrayBuffer = await audioBlob.arrayBuffer(); const base64Audio = btoa( new Uint8Array(arrayBuffer).reduce( (data, byte) => data + String.fromCharCode(byte), "" ) ); ws.send({ type: "session", message: { type: "audio_chunk", audio: base64Audio, format: format, isLast: true, }, }); console.log( `[App] Sent audio: ${audioBlob.size} bytes, format: ${format}` ); } else { console.log("[App] Starting recording..."); // Stop any currently playing audio when starting to record audioPlayerRef.current.stop(); setIsPlayingAudio(false); await recorder.start(); setIsRecording(true); } } catch (error: any) { console.error("[App] Recording error:", error); addLog("error", `Failed to record audio: ${error.message}`); setIsRecording(false); } }; const handleCancel = async () => { console.log("[App] Cancelling operations..."); // Stop audio playback audioPlayerRef.current.stop(); setIsPlayingAudio(false); // Clear streaming state setCurrentAssistantMessage(""); // Reset processing state setIsProcessingAudio(false); // Send abort request to server ws.send({ type: "session", message: { type: "abort_request", }, }); addLog("info", "Operations cancelled"); }; // Compute if we're processing an already sent request (not including recording) const isInProgress = isProcessingAudio || isPlayingAudio || currentAssistantMessage.length > 0; const handleButtonClick = async (e: React.MouseEvent) => { e.preventDefault(); if (isRecording) { // Stop recording and send the audio await handleToggleRecording(); } else if (isInProgress) { // Cancel processing/playback of already sent request await handleCancel(); } else if (userInput.trim()) { handleSendMessage(); } else { // Start recording await handleToggleRecording(); } }; const handleKeyDown = (e: React.KeyboardEvent) => { // Detect if device is desktop (not mobile/tablet) const isDesktop = window.matchMedia("(pointer: fine)").matches; if (e.key === "Enter") { if (isDesktop && !e.shiftKey && userInput.trim()) { // Desktop: plain Enter sends message e.preventDefault(); handleSendMessage(); } // Desktop: Shift+Enter creates new line (default behavior) // Mobile: Enter creates new line (default behavior) } }; const handleCloseArtifact = () => { setCurrentArtifact(null); }; const handleOpenArtifact = (artifactId: string) => { const artifact = artifacts.get(artifactId); if (artifact) { setCurrentArtifact(artifact); } }; const handleSelectConversation = (conversationId: string | null) => { if (conversationId === null) { // New conversation clearCurrentConversationId(); } else { // Load existing conversation setCurrentConversationId(conversationId); } // Update state to trigger WebSocket reconnection setSelectedConversationId(conversationId); // Clear UI state setLogs([ { id: "1", timestamp: Date.now(), type: "system", message: conversationId ? "Loading conversation..." : "Starting new conversation...", }, ]); setUserInput(""); setCurrentAssistantMessage(""); setArtifacts(new Map()); setCurrentArtifact(null); }; const handleSelectProcess = (id: string, type: "terminal" | "agent") => { setActiveProcessId(id); setActiveView(type); }; const handleBackToOrchestrator = () => { setActiveView("orchestrator"); setActiveProcessId(null); }; const handleKillAgent = async (agentId: string) => { // In a full implementation, this would send a message to the server // For now, just log console.log("[App] Kill agent:", agentId); addLog("info", `Killing agent ${agentId.substring(0, 8)}...`); }; const handleCancelAgent = async (agentId: string) => { // In a full implementation, this would send a message to the server // For now, just log console.log("[App] Cancel agent:", agentId); addLog("info", `Cancelling agent ${agentId.substring(0, 8)}...`); }; const handleToggleRealtimeMode = async () => { if (!ws.isConnected) return; try { if (isRealtimeMode) { // Stop realtime mode if (vadRef.current) { vadRef.current.pause(); } setIsRealtimeMode(false); setIsSpeechDetected(false); addLog("info", "Realtime mode stopped"); } else { // Start realtime mode setIsVADLoading(true); addLog("info", "Starting realtime mode..."); // Create VAD instance if it doesn't exist if (!vadRef.current) { vadRef.current = createRealtimeVAD({ onModelLoading: () => { console.log("[App] VAD model loading..."); setIsVADLoading(true); }, onModelLoaded: () => { console.log("[App] VAD model loaded"); setIsVADLoading(false); }, onSpeechStart: () => { console.log("[App] Speech detected!"); setIsSpeechDetected(true); // Pause playback when speech is detected // Will resume if false positive, or stop completely if real speech audioPlayerRef.current.pause(); }, onSpeechEnd: async (audioData: Float32Array) => { console.log("[App] Speech ended, processing..."); setIsSpeechDetected(false); setIsProcessingAudio(true); try { // Convert Float32Array to audio blob const audioBlob = float32ArrayToBlob(audioData, 16000); console.log(`[App] Converted to blob: ${audioBlob.size} bytes`); // Send to server const arrayBuffer = await audioBlob.arrayBuffer(); const base64Audio = btoa( new Uint8Array(arrayBuffer).reduce( (data, byte) => data + String.fromCharCode(byte), "" ) ); ws.send({ type: "session", message: { type: "audio_chunk", audio: base64Audio, format: audioBlob.type, isLast: true, }, }); console.log(`[App] Sent realtime audio to server`); } catch (error: any) { console.error("[App] Failed to process VAD audio:", error); addLog("error", `Failed to process audio: ${error.message}`); setIsProcessingAudio(false); } }, onVADMisfire: () => { console.log("[App] VAD misfire - resuming playback"); setIsSpeechDetected(false); // Resume playback since it was a false positive audioPlayerRef.current.resume(); }, onError: (error: Error) => { console.error("[App] VAD error:", error); addLog("error", `VAD error: ${error.message}`); setIsVADLoading(false); setIsRealtimeMode(false); }, }); } // Start VAD await vadRef.current.start(); setIsRealtimeMode(true); setIsVADLoading(false); addLog("success", "Realtime mode started - speak anytime!"); } } catch (error: any) { console.error("[App] Realtime mode error:", error); addLog("error", `Failed to toggle realtime mode: ${error.message}`); setIsRealtimeMode(false); setIsVADLoading(false); } }; return (
{isPlayingAudio && (
)}
{activeView === "orchestrator" && ( <>
{logs.map((log) => { if (log.type === "tool_call") { return ( ); } if (log.type === "artifact") { return (
handleOpenArtifact(log.artifactId)} role="button" tabIndex={0} onKeyDown={(e) => { if (e.key === "Enter" || e.key === " ") { handleOpenArtifact(log.artifactId); } }} > 📋 {log.artifactType}: {log.title}
); } return (
{log.message} {log.metadata && (
Details
{JSON.stringify(log.metadata, null, 2)}
)}
); })} {currentAssistantMessage && (
{currentAssistantMessage} ...
)}