import { useState, useEffect, useRef } from "react"; import { Mic, Send, Radio } from "lucide-react"; import { useWebSocket } from "./hooks/useWebSocket"; import { createAudioPlayer } from "./lib/audio-playback"; import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture"; import { createRealtimeVAD, float32ArrayToBlob, type RealtimeVAD, } from "./lib/audio-realtime"; import { ToolCallCard } from "./components/ToolCallCard"; import { ArtifactDrawer, type Artifact } from "./components/ArtifactDrawer"; import "./App.css"; type LogEntry = | { type: "system" | "info" | "success" | "error" | "user" | "assistant"; id: string; timestamp: number; message: string; metadata?: Record; } | { type: "tool_call"; id: string; timestamp: number; toolName: string; args: any; result?: any; error?: any; status: "executing" | "completed" | "failed"; } | { type: "artifact"; id: string; timestamp: number; artifactId: string; artifactType: string; title: string; }; function App() { const [logs, setLogs] = useState([ { id: "1", timestamp: Date.now(), type: "system", message: "System initialized", }, ]); const [userInput, setUserInput] = useState(""); const [currentAssistantMessage, setCurrentAssistantMessage] = useState(""); const [isProcessingAudio, setIsProcessingAudio] = useState(false); const [isPlayingAudio, setIsPlayingAudio] = useState(false); const [isRecording, setIsRecording] = useState(false); const [isRealtimeMode, setIsRealtimeMode] = useState(false); const [isSpeechDetected, setIsSpeechDetected] = useState(false); const [isVADLoading, setIsVADLoading] = useState(false); const [currentArtifact, setCurrentArtifact] = useState(null); const [artifacts, setArtifacts] = useState>(new Map()); const logEndRef = useRef(null); const audioPlayerRef = useRef(createAudioPlayer()); const recorderRef = useRef(null); const vadRef = useRef(null); const textareaRef = useRef(null); // WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod const wsUrl = `${window.location.protocol === "https:" ? "wss" : "ws"}://${ window.location.host }/ws`; const ws = useWebSocket(wsUrl); useEffect(() => { recorderRef.current = createAudioRecorder(); // Cleanup VAD on unmount return () => { if (vadRef.current) { vadRef.current.destroy(); } }; }, []); useEffect(() => { if (textareaRef.current) { textareaRef.current.style.height = "auto"; const newHeight = Math.max(textareaRef.current.scrollHeight, 44); textareaRef.current.style.height = `${newHeight}px`; } }, [userInput]); const addLog = ( type: "system" | "info" | "success" | "error" | "user" | "assistant", message: string, metadata?: Record ) => { setLogs((prev) => [ ...prev, { id: Date.now().toString(), timestamp: Date.now(), type, message, metadata, }, ]); }; useEffect(() => { // Listen for status messages const unsubStatus = ws.on("status", (message) => { if (message.type !== "status") return; const msg = "message" in message.payload ? String(message.payload.message) : undefined; addLog("info", msg || `Status: ${message.payload.status}`); }); // Listen for activity log messages const unsubActivity = ws.on("activity_log", (message) => { if (message.type !== "activity_log") return; const data = message.payload; // Handle tool calls if (data.type === "tool_call" && data.metadata) { const { toolCallId, toolName, arguments: args, } = data.metadata as { toolCallId: string; toolName: string; arguments: unknown; }; setLogs((prev) => [ ...prev, { type: "tool_call", id: toolCallId, timestamp: Date.now(), toolName, args, status: "executing", }, ]); return; } if (data.type === "tool_result" && data.metadata) { const { toolCallId, result } = data.metadata as { toolCallId: string; result: unknown; }; setLogs((prev) => prev.map((log) => log.type === "tool_call" && log.id === toolCallId ? { ...log, result, status: "completed" as const } : log ) ); return; } // Handle tool errors - update tool call status to failed if ( data.type === "error" && data.metadata && "toolCallId" in data.metadata ) { const { toolCallId, error } = data.metadata as { toolCallId: string; error: unknown; }; setLogs((prev) => prev.map((log) => log.type === "tool_call" && log.id === toolCallId ? { ...log, error, status: "failed" as const } : log ) ); // Don't return - still add the error message to the log } // Map activity log types to UI log types let logType: | "system" | "info" | "success" | "error" | "user" | "assistant" = "info"; if (data.type === "transcript") logType = "user"; else if (data.type === "assistant") logType = "assistant"; else if (data.type === "error") logType = "error"; addLog(logType, data.content, data.metadata); // Clear streaming state when assistant segment is received if (data.type === "assistant") { setCurrentAssistantMessage(""); } }); // Listen for streaming assistant chunks const unsubChunk = ws.on("assistant_chunk", (message) => { if (message.type !== "assistant_chunk") return; setCurrentAssistantMessage((prev) => prev + message.payload.chunk); }); // Listen for transcription results const unsubTranscription = ws.on("transcription_result", (message) => { if (message.type !== "transcription_result") return; setIsProcessingAudio(false); const transcriptText = message.payload.text.trim(); if (!transcriptText) { // Empty transcription - false positive, resume playback console.log( "[App] Empty transcription (false positive) - resuming playback" ); audioPlayerRef.current.resume(); } else { // Has content - real speech detected // Server will handle abort, just stop playback on client console.log("[App] Transcription received - stopping playback"); audioPlayerRef.current.stop(); setIsPlayingAudio(false); setCurrentAssistantMessage(""); } }); // Listen for artifacts const unsubArtifact = ws.on("artifact", (message) => { if (message.type !== "artifact") return; const artifact = message.payload; // Add to artifacts map setArtifacts((prev) => { const newMap = new Map(prev); newMap.set(artifact.id, artifact); return newMap; }); // Add as clickable log entry setLogs((prev) => [ ...prev, { type: "artifact" as const, id: Date.now().toString(), timestamp: Date.now(), artifactId: artifact.id, artifactType: artifact.type, title: artifact.title, }, ]); // Show drawer by default setCurrentArtifact(artifact); }); // Listen for audio output (TTS) const unsubAudioOutput = ws.on("audio_output", async (message) => { if (message.type !== "audio_output") return; const data = message.payload; try { setIsPlayingAudio(true); // Decode base64 audio const binaryString = atob(data.audio); const bytes = new Uint8Array(binaryString.length); for (let i = 0; i < binaryString.length; i++) { bytes[i] = binaryString.charCodeAt(i); } // Create blob with correct mime type const mimeType = data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`; const audioBlob = new Blob([bytes], { type: mimeType }); // Play audio await audioPlayerRef.current.play(audioBlob); // Send confirmation back to server ws.send({ type: "session", message: { type: "audio_played", id: data.id, }, }); setIsPlayingAudio(false); } catch (error: any) { console.error("[App] Audio playback error:", error); addLog("error", `Audio playback failed: ${error.message}`); setIsPlayingAudio(false); } }); return () => { unsubStatus(); unsubActivity(); unsubChunk(); unsubTranscription(); unsubArtifact(); unsubAudioOutput(); }; }, [ws]); useEffect(() => { if (ws.isConnected) { addLog("success", "WebSocket connected"); } else { addLog("system", "WebSocket disconnected"); } }, [ws.isConnected]); // Auto-scroll to bottom when new logs are added useEffect(() => { logEndRef.current?.scrollIntoView({ behavior: "smooth" }); }, [logs, currentAssistantMessage]); const handleSendMessage = () => { if (!userInput.trim() || !ws.isConnected) return; // Stop any currently playing audio and clear the queue audioPlayerRef.current.stop(); setIsPlayingAudio(false); // Send message to server ws.sendUserMessage(userInput); // Clear input and reset streaming state setUserInput(""); setCurrentAssistantMessage(""); }; const handleToggleRecording = async () => { const recorder = recorderRef.current; if (!recorder || !ws.isConnected) return; try { if (isRecording) { console.log("[App] Stopping recording..."); const audioBlob = await recorder.stop(); setIsRecording(false); const format = audioBlob.type || "audio/webm"; console.log( `[App] Recording complete: ${audioBlob.size} bytes, format: ${format}` ); setIsProcessingAudio(true); addLog("info", "Sending audio to server..."); const arrayBuffer = await audioBlob.arrayBuffer(); const base64Audio = btoa( new Uint8Array(arrayBuffer).reduce( (data, byte) => data + String.fromCharCode(byte), "" ) ); ws.send({ type: "session", message: { type: "audio_chunk", audio: base64Audio, format: format, isLast: true, }, }); console.log( `[App] Sent audio: ${audioBlob.size} bytes, format: ${format}` ); } else { console.log("[App] Starting recording..."); // Stop any currently playing audio when starting to record audioPlayerRef.current.stop(); setIsPlayingAudio(false); await recorder.start(); setIsRecording(true); } } catch (error: any) { console.error("[App] Recording error:", error); addLog("error", `Failed to record audio: ${error.message}`); setIsRecording(false); } }; const handleButtonClick = async (e: React.MouseEvent) => { e.preventDefault(); if (userInput.trim()) { handleSendMessage(); } else { await handleToggleRecording(); } }; const handleKeyDown = (e: React.KeyboardEvent) => { // Detect if device is desktop (not mobile/tablet) const isDesktop = window.matchMedia("(pointer: fine)").matches; if (e.key === "Enter") { if (isDesktop && !e.shiftKey && userInput.trim()) { // Desktop: plain Enter sends message e.preventDefault(); handleSendMessage(); } // Desktop: Shift+Enter creates new line (default behavior) // Mobile: Enter creates new line (default behavior) } }; const handleCloseArtifact = () => { setCurrentArtifact(null); }; const handleOpenArtifact = (artifactId: string) => { const artifact = artifacts.get(artifactId); if (artifact) { setCurrentArtifact(artifact); } }; const handleToggleRealtimeMode = async () => { if (!ws.isConnected) return; try { if (isRealtimeMode) { // Stop realtime mode if (vadRef.current) { vadRef.current.pause(); } setIsRealtimeMode(false); setIsSpeechDetected(false); addLog("info", "Realtime mode stopped"); } else { // Start realtime mode setIsVADLoading(true); addLog("info", "Starting realtime mode..."); // Create VAD instance if it doesn't exist if (!vadRef.current) { vadRef.current = createRealtimeVAD({ onModelLoading: () => { console.log("[App] VAD model loading..."); setIsVADLoading(true); }, onModelLoaded: () => { console.log("[App] VAD model loaded"); setIsVADLoading(false); }, onSpeechStart: () => { console.log("[App] Speech detected!"); setIsSpeechDetected(true); // Pause playback when speech is detected // Will resume if false positive, or stop completely if real speech audioPlayerRef.current.pause(); }, onSpeechEnd: async (audioData: Float32Array) => { console.log("[App] Speech ended, processing..."); setIsSpeechDetected(false); setIsProcessingAudio(true); try { // Convert Float32Array to audio blob const audioBlob = float32ArrayToBlob(audioData, 16000); console.log(`[App] Converted to blob: ${audioBlob.size} bytes`); // Send to server const arrayBuffer = await audioBlob.arrayBuffer(); const base64Audio = btoa( new Uint8Array(arrayBuffer).reduce( (data, byte) => data + String.fromCharCode(byte), "" ) ); ws.send({ type: "session", message: { type: "audio_chunk", audio: base64Audio, format: audioBlob.type, isLast: true, }, }); console.log(`[App] Sent realtime audio to server`); } catch (error: any) { console.error("[App] Failed to process VAD audio:", error); addLog("error", `Failed to process audio: ${error.message}`); setIsProcessingAudio(false); } }, onVADMisfire: () => { console.log("[App] VAD misfire - resuming playback"); setIsSpeechDetected(false); // Resume playback since it was a false positive audioPlayerRef.current.resume(); }, onError: (error: Error) => { console.error("[App] VAD error:", error); addLog("error", `VAD error: ${error.message}`); setIsVADLoading(false); setIsRealtimeMode(false); }, }); } // Start VAD await vadRef.current.start(); setIsRealtimeMode(true); setIsVADLoading(false); addLog("success", "Realtime mode started - speak anytime!"); } } catch (error: any) { console.error("[App] Realtime mode error:", error); addLog("error", `Failed to toggle realtime mode: ${error.message}`); setIsRealtimeMode(false); setIsVADLoading(false); } }; return (

Voice Assistant

{isPlayingAudio && (
)}
{ws.isConnected ? "connected" : "disconnected"}
{logs.map((log) => { if (log.type === "tool_call") { return ( ); } if (log.type === "artifact") { return (
handleOpenArtifact(log.artifactId)} role="button" tabIndex={0} onKeyDown={(e) => { if (e.key === "Enter" || e.key === " ") { handleOpenArtifact(log.artifactId); } }} > 📋 {log.artifactType}: {log.title}
); } return (
{log.message} {log.metadata && (
Details
{JSON.stringify(log.metadata, null, 2)}
)}
); })} {currentAssistantMessage && (
{currentAssistantMessage} ...
)}