paseo/packages/voice-assistant/src/ui/App.tsx

import { useState, useEffect, useRef } from "react";
import { Mic, Send, Radio, X } from "lucide-react";
import { useWebSocket } from "./hooks/useWebSocket";
import { createAudioPlayer } from "./lib/audio-playback";
import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture";
import {
  createRealtimeVAD,
  float32ArrayToBlob,
  type RealtimeVAD,
} from "./lib/audio-realtime";
import { ToolCallCard } from "./components/ToolCallCard";
import { ArtifactDrawer, type Artifact } from "./components/ArtifactDrawer";
import { ConversationSelector } from "./components/ConversationSelector";
import { ActiveProcesses } from "./components/ActiveProcesses";
import { AgentStreamView } from "./components/AgentStreamView";
import {
  getCurrentConversationId,
  setCurrentConversationId,
  clearCurrentConversationId,
} from "./lib/conversation-storage";
import type { AgentStatus } from "../server/acp/types.js";
import type { SessionNotification } from "@agentclientprotocol/sdk";
import "./App.css";

type LogEntry =
  | {
      type: "system" | "info" | "success" | "error" | "user" | "assistant";
      id: string;
      timestamp: number;
      message: string;
      metadata?: Record<string, unknown>;
    }
  | {
      type: "tool_call";
      id: string;
      timestamp: number;
      toolName: string;
      args: any;
      result?: any;
      error?: any;
      status: "executing" | "completed" | "failed";
    }
  | {
      type: "artifact";
      id: string;
      timestamp: number;
      artifactId: string;
      artifactType: string;
      title: string;
    };

function App() {
  const [logs, setLogs] = useState<LogEntry[]>([
    {
      id: "1",
      timestamp: Date.now(),
      type: "system",
      message: "System initialized",
    },
  ]);
  const [userInput, setUserInput] = useState("");
  const [currentAssistantMessage, setCurrentAssistantMessage] = useState("");
  const [isProcessingAudio, setIsProcessingAudio] = useState(false);
  const [isPlayingAudio, setIsPlayingAudio] = useState(false);
  const [isRecording, setIsRecording] = useState(false);
  const [isRealtimeMode, setIsRealtimeMode] = useState(false);
  const [isSpeechDetected, setIsSpeechDetected] = useState(false);
  const [isVADLoading, setIsVADLoading] = useState(false);
  const [currentArtifact, setCurrentArtifact] = useState<Artifact | null>(null);
  const [artifacts, setArtifacts] = useState<Map<string, Artifact>>(new Map());
  const [selectedConversationId, setSelectedConversationId] = useState<
    string | null
  >(() => getCurrentConversationId());
  const [agents, setAgents] = useState<
    Map<
      string,
      { id: string; status: AgentStatus; createdAt: Date; type: "claude" }
    >
  >(new Map());
  const [agentUpdates, setAgentUpdates] = useState<
    Map<string, Array<{ timestamp: Date; notification: SessionNotification }>>
  >(new Map());
  const [activeView, setActiveView] = useState<
    "orchestrator" | "terminal" | "agent"
  >("orchestrator");
  const [activeProcessId, setActiveProcessId] = useState<string | null>(null);
  const logEndRef = useRef<HTMLDivElement>(null);
  const audioPlayerRef = useRef(createAudioPlayer());
  const recorderRef = useRef<AudioRecorder | null>(null);
  const vadRef = useRef<RealtimeVAD | null>(null);
  const textareaRef = useRef<HTMLTextAreaElement>(null);

  // WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod
  const wsUrl = `${window.location.protocol === "https:" ? "wss" : "ws"}://${
    window.location.host
  }/ws`;

  const ws = useWebSocket(wsUrl, selectedConversationId);

  useEffect(() => {
    recorderRef.current = createAudioRecorder();

    // Cleanup VAD on unmount
    return () => {
      if (vadRef.current) {
        vadRef.current.destroy();
      }
    };
  }, []);

  useEffect(() => {
    if (textareaRef.current) {
      textareaRef.current.style.height = "auto";
      const newHeight = Math.max(textareaRef.current.scrollHeight, 44);
      textareaRef.current.style.height = `${newHeight}px`;
    }
  }, [userInput]);

  const addLog = (
    type: "system" | "info" | "success" | "error" | "user" | "assistant",
    message: string,
    metadata?: Record<string, unknown>
  ) => {
    setLogs((prev) => [
      ...prev,
      {
        id: Date.now().toString(),
        timestamp: Date.now(),
        type,
        message,
        metadata,
      },
    ]);
  };

  useEffect(() => {
    // Listen for status messages
    const unsubStatus = ws.on("status", (message) => {
      if (message.type !== "status") return;
      const msg =
        "message" in message.payload
          ? String(message.payload.message)
          : undefined;
      addLog("info", msg || `Status: ${message.payload.status}`);
    });

    // Listen for activity log messages
    const unsubActivity = ws.on("activity_log", (message) => {
      if (message.type !== "activity_log") return;
      const data = message.payload;

      // Handle tool calls
      if (data.type === "tool_call" && data.metadata) {
        const {
          toolCallId,
          toolName,
          arguments: args,
        } = data.metadata as {
          toolCallId: string;
          toolName: string;
          arguments: unknown;
        };

        setLogs((prev) => [
          ...prev,
          {
            type: "tool_call",
            id: toolCallId,
            timestamp: Date.now(),
            toolName,
            args,
            status: "executing",
          },
        ]);
        return;
      }

      if (data.type === "tool_result" && data.metadata) {
        const { toolCallId, result } = data.metadata as {
          toolCallId: string;
          result: unknown;
        };

        setLogs((prev) =>
          prev.map((log) =>
            log.type === "tool_call" && log.id === toolCallId
              ? { ...log, result, status: "completed" as const }
              : log
          )
        );
        return;
      }

      // Handle tool errors - update tool call status to failed
      if (
        data.type === "error" &&
        data.metadata &&
        "toolCallId" in data.metadata
      ) {
        const { toolCallId, error } = data.metadata as {
          toolCallId: string;
          error: unknown;
        };

        setLogs((prev) =>
          prev.map((log) =>
            log.type === "tool_call" && log.id === toolCallId
              ? { ...log, error, status: "failed" as const }
              : log
          )
        );
        // Don't return - still add the error message to the log
      }

      // Map activity log types to UI log types
      let logType:
        | "system"
        | "info"
        | "success"
        | "error"
        | "user"
        | "assistant" = "info";
      if (data.type === "transcript") logType = "user";
      else if (data.type === "assistant") logType = "assistant";
      else if (data.type === "error") logType = "error";

      addLog(logType, data.content, data.metadata);

      // Clear streaming state when assistant segment is received
      if (data.type === "assistant") {
        setCurrentAssistantMessage("");
      }
    });

    // Listen for streaming assistant chunks
    const unsubChunk = ws.on("assistant_chunk", (message) => {
      if (message.type !== "assistant_chunk") return;
      setCurrentAssistantMessage((prev) => prev + message.payload.chunk);
    });

    // Listen for transcription results
    const unsubTranscription = ws.on("transcription_result", (message) => {
      if (message.type !== "transcription_result") return;

      setIsProcessingAudio(false);

      const transcriptText = message.payload.text.trim();

      if (!transcriptText) {
        // Empty transcription - false positive, resume playback
        console.log(
          "[App] Empty transcription (false positive) - resuming playback"
        );
        audioPlayerRef.current.resume();
      } else {
        // Has content - real speech detected
        // Server will handle abort, just stop playback on client
        console.log("[App] Transcription received - stopping playback");
        audioPlayerRef.current.stop();
        setIsPlayingAudio(false);
        setCurrentAssistantMessage("");
      }
    });

    // Listen for conversation loaded confirmation
    const unsubConversationLoaded = ws.on("conversation_loaded", (message) => {
      if (message.type !== "conversation_loaded") return;
      const { conversationId, messageCount } = message.payload;

      // Save to localStorage for persistence
      setCurrentConversationId(conversationId);

      addLog(
        "success",
        `Loaded conversation with ${messageCount} messages`
      );
    });

    // Listen for artifacts
    const unsubArtifact = ws.on("artifact", (message) => {
      if (message.type !== "artifact") return;
      const artifact = message.payload;

      // Add to artifacts map
      setArtifacts((prev) => {
        const newMap = new Map(prev);
        newMap.set(artifact.id, artifact);
        return newMap;
      });

      // Add as clickable log entry
      setLogs((prev) => [
        ...prev,
        {
          type: "artifact" as const,
          id: Date.now().toString(),
          timestamp: Date.now(),
          artifactId: artifact.id,
          artifactType: artifact.type,
          title: artifact.title,
        },
      ]);

      // Show drawer by default
      setCurrentArtifact(artifact);
    });

    // Listen for audio output (TTS)
    const unsubAudioOutput = ws.on("audio_output", async (message) => {
      if (message.type !== "audio_output") return;
      const data = message.payload;

      try {
        setIsPlayingAudio(true);

        // Decode base64 audio
        const binaryString = atob(data.audio);
        const bytes = new Uint8Array(binaryString.length);
        for (let i = 0; i < binaryString.length; i++) {
          bytes[i] = binaryString.charCodeAt(i);
        }

        // Create blob with correct mime type
        const mimeType =
          data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`;
        const audioBlob = new Blob([bytes], { type: mimeType });

        // Play audio
        await audioPlayerRef.current.play(audioBlob);

        // Send confirmation back to server
        ws.send({
          type: "session",
          message: {
            type: "audio_played",
            id: data.id,
          },
        });

        setIsPlayingAudio(false);
      } catch (error: any) {
        console.error("[App] Audio playback error:", error);
        addLog("error", `Audio playback failed: ${error.message}`);
        setIsPlayingAudio(false);
      }
    });

    // Listen for agent created
    const unsubAgentCreated = ws.on("agent_created", (message) => {
      if (message.type !== "agent_created") return;
      const { agentId, status, type } = message.payload;

      setAgents((prev) => {
        const newMap = new Map(prev);
        newMap.set(agentId, {
          id: agentId,
          status: status as AgentStatus,
          createdAt: new Date(),
          type,
        });
        return newMap;
      });

      addLog("success", `Agent ${agentId.substring(0, 8)} created`);
    });

    // Listen for agent updates
    const unsubAgentUpdate = ws.on("agent_update", (message) => {
      if (message.type !== "agent_update") return;
      const { agentId, timestamp, notification } = message.payload;

      setAgentUpdates((prev) => {
        const newMap = new Map(prev);
        const updates = newMap.get(agentId) || [];
        newMap.set(agentId, [
          ...updates,
          {
            timestamp: new Date(timestamp),
            notification: notification as SessionNotification,
          },
        ]);
        return newMap;
      });
    });

    // Listen for agent status changes
    const unsubAgentStatus = ws.on("agent_status", (message) => {
      if (message.type !== "agent_status") return;
      const { agentId, status } = message.payload;

      setAgents((prev) => {
        const newMap = new Map(prev);
        const agent = newMap.get(agentId);
        if (agent) {
          newMap.set(agentId, {
            ...agent,
            status: status as AgentStatus,
          });
        }
        return newMap;
      });

      addLog("info", `Agent ${agentId.substring(0, 8)}: ${status}`);
    });

    return () => {
      unsubStatus();
      unsubActivity();
      unsubChunk();
      unsubTranscription();
      unsubConversationLoaded();
      unsubArtifact();
      unsubAudioOutput();
      unsubAgentCreated();
      unsubAgentUpdate();
      unsubAgentStatus();
    };
  }, [ws]);

  useEffect(() => {
    if (ws.isConnected) {
      addLog("success", "WebSocket connected");
    } else {
      addLog("system", "WebSocket disconnected");
    }
  }, [ws.isConnected]);

  // Auto-scroll to bottom when new logs are added
  useEffect(() => {
    logEndRef.current?.scrollIntoView({ behavior: "smooth" });
  }, [logs, currentAssistantMessage]);

  const handleSendMessage = () => {
    if (!userInput.trim() || !ws.isConnected) return;

    // Stop any currently playing audio and clear the queue
    audioPlayerRef.current.stop();
    setIsPlayingAudio(false);

    // Send message to server
    ws.sendUserMessage(userInput);

    // Clear input and reset streaming state
    setUserInput("");
    setCurrentAssistantMessage("");
  };

  const handleToggleRecording = async () => {
    const recorder = recorderRef.current;
    if (!recorder || !ws.isConnected) return;

    try {
      if (isRecording) {
        console.log("[App] Stopping recording...");
        const audioBlob = await recorder.stop();
        setIsRecording(false);

        const format = audioBlob.type || "audio/webm";
        console.log(
          `[App] Recording complete: ${audioBlob.size} bytes, format: ${format}`
        );

        setIsProcessingAudio(true);
        addLog("info", "Sending audio to server...");

        const arrayBuffer = await audioBlob.arrayBuffer();
        const base64Audio = btoa(
          new Uint8Array(arrayBuffer).reduce(
            (data, byte) => data + String.fromCharCode(byte),
            ""
          )
        );

        ws.send({
          type: "session",
          message: {
            type: "audio_chunk",
            audio: base64Audio,
            format: format,
            isLast: true,
          },
        });

        console.log(
          `[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`
        );
      } else {
        console.log("[App] Starting recording...");

        // Stop any currently playing audio when starting to record
        audioPlayerRef.current.stop();
        setIsPlayingAudio(false);

        await recorder.start();
        setIsRecording(true);
      }
    } catch (error: any) {
      console.error("[App] Recording error:", error);
      addLog("error", `Failed to record audio: ${error.message}`);
      setIsRecording(false);
    }
  };

  const handleCancel = async () => {
    console.log("[App] Cancelling operations...");

    // Stop audio playback
    audioPlayerRef.current.stop();
    setIsPlayingAudio(false);

    // Clear streaming state
    setCurrentAssistantMessage("");

    // Reset processing state
    setIsProcessingAudio(false);

    // Send abort request to server
    ws.send({
      type: "session",
      message: {
        type: "abort_request",
      },
    });

    addLog("info", "Operations cancelled");
  };

  // Compute if we're processing an already sent request (not including recording)
  const isInProgress =
    isProcessingAudio || isPlayingAudio || currentAssistantMessage.length > 0;

  const handleButtonClick = async (e: React.MouseEvent<HTMLButtonElement>) => {
    e.preventDefault();

    if (isRecording) {
      // Stop recording and send the audio
      await handleToggleRecording();
    } else if (isInProgress) {
      // Cancel processing/playback of already sent request
      await handleCancel();
    } else if (userInput.trim()) {
      handleSendMessage();
    } else {
      // Start recording
      await handleToggleRecording();
    }
  };

  const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
    // Detect if device is desktop (not mobile/tablet)
    const isDesktop = window.matchMedia("(pointer: fine)").matches;

    if (e.key === "Enter") {
      if (isDesktop && !e.shiftKey && userInput.trim()) {
        // Desktop: plain Enter sends message
        e.preventDefault();
        handleSendMessage();
      }
      // Desktop: Shift+Enter creates new line (default behavior)
      // Mobile: Enter creates new line (default behavior)
    }
  };

  const handleCloseArtifact = () => {
    setCurrentArtifact(null);
  };

  const handleOpenArtifact = (artifactId: string) => {
    const artifact = artifacts.get(artifactId);
    if (artifact) {
      setCurrentArtifact(artifact);
    }
  };

  const handleSelectConversation = (conversationId: string | null) => {
    if (conversationId === null) {
      // New conversation
      clearCurrentConversationId();
    } else {
      // Load existing conversation
      setCurrentConversationId(conversationId);
    }

    // Update state to trigger WebSocket reconnection
    setSelectedConversationId(conversationId);

    // Clear UI state
    setLogs([
      {
        id: "1",
        timestamp: Date.now(),
        type: "system",
        message: conversationId
          ? "Loading conversation..."
          : "Starting new conversation...",
      },
    ]);
    setUserInput("");
    setCurrentAssistantMessage("");
    setArtifacts(new Map());
    setCurrentArtifact(null);
  };

  const handleSelectProcess = (id: string, type: "terminal" | "agent") => {
    setActiveProcessId(id);
    setActiveView(type);
  };

  const handleBackToOrchestrator = () => {
    setActiveView("orchestrator");
    setActiveProcessId(null);
  };

  const handleKillAgent = async (agentId: string) => {
    // In a full implementation, this would send a message to the server
    // For now, just log
    console.log("[App] Kill agent:", agentId);
    addLog("info", `Killing agent ${agentId.substring(0, 8)}...`);
  };

  const handleCancelAgent = async (agentId: string) => {
    // In a full implementation, this would send a message to the server
    // For now, just log
    console.log("[App] Cancel agent:", agentId);
    addLog("info", `Cancelling agent ${agentId.substring(0, 8)}...`);
  };

  const handleToggleRealtimeMode = async () => {
    if (!ws.isConnected) return;

    try {
      if (isRealtimeMode) {
        // Stop realtime mode
        if (vadRef.current) {
          vadRef.current.pause();
        }
        setIsRealtimeMode(false);
        setIsSpeechDetected(false);
        addLog("info", "Realtime mode stopped");
      } else {
        // Start realtime mode
        setIsVADLoading(true);
        addLog("info", "Starting realtime mode...");

        // Create VAD instance if it doesn't exist
        if (!vadRef.current) {
          vadRef.current = createRealtimeVAD({
            onModelLoading: () => {
              console.log("[App] VAD model loading...");
              setIsVADLoading(true);
            },
            onModelLoaded: () => {
              console.log("[App] VAD model loaded");
              setIsVADLoading(false);
            },
            onSpeechStart: () => {
              console.log("[App] Speech detected!");
              setIsSpeechDetected(true);

              // Pause playback when speech is detected
              // Will resume if false positive, or stop completely if real speech
              audioPlayerRef.current.pause();
            },
            onSpeechEnd: async (audioData: Float32Array) => {
              console.log("[App] Speech ended, processing...");
              setIsSpeechDetected(false);
              setIsProcessingAudio(true);

              try {
                // Convert Float32Array to audio blob
                const audioBlob = float32ArrayToBlob(audioData, 16000);
                console.log(`[App] Converted to blob: ${audioBlob.size} bytes`);

                // Send to server
                const arrayBuffer = await audioBlob.arrayBuffer();
                const base64Audio = btoa(
                  new Uint8Array(arrayBuffer).reduce(
                    (data, byte) => data + String.fromCharCode(byte),
                    ""
                  )
                );

                ws.send({
                  type: "session",
                  message: {
                    type: "audio_chunk",
                    audio: base64Audio,
                    format: audioBlob.type,
                    isLast: true,
                  },
                });

                console.log(`[App] Sent realtime audio to server`);
              } catch (error: any) {
                console.error("[App] Failed to process VAD audio:", error);
                addLog("error", `Failed to process audio: ${error.message}`);
                setIsProcessingAudio(false);
              }
            },
            onVADMisfire: () => {
              console.log("[App] VAD misfire - resuming playback");
              setIsSpeechDetected(false);

              // Resume playback since it was a false positive
              audioPlayerRef.current.resume();
            },
            onError: (error: Error) => {
              console.error("[App] VAD error:", error);
              addLog("error", `VAD error: ${error.message}`);
              setIsVADLoading(false);
              setIsRealtimeMode(false);
            },
          });
        }

        // Start VAD
        await vadRef.current.start();
        setIsRealtimeMode(true);
        setIsVADLoading(false);
        addLog("success", "Realtime mode started - speak anytime!");
      }
    } catch (error: any) {
      console.error("[App] Realtime mode error:", error);
      addLog("error", `Failed to toggle realtime mode: ${error.message}`);
      setIsRealtimeMode(false);
      setIsVADLoading(false);
    }
  };

  return (
    <div className="app">
      <header className="header">
        <ConversationSelector
          currentConversationId={ws.conversationId}
          onSelectConversation={handleSelectConversation}
        />
        {isPlayingAudio && (
          <div className="audio-playing-indicator" title="Playing audio" />
        )}
      </header>

      <main className="main">
        <div className="chat-interface">
          {activeView === "orchestrator" && (
            <>
              <div className="activity-log">
                <div className="log-entries">
                  {logs.map((log) => {
                    if (log.type === "tool_call") {
                      return (
                        <ToolCallCard
                          key={log.id}
                          toolName={log.toolName}
                          args={log.args}
                          result={log.result}
                          error={log.error}
                          status={log.status}
                        />
                      );
                    }

                    if (log.type === "artifact") {
                      return (
                        <div
                          key={log.id}
                          className="log-entry artifact clickable"
                          onClick={() => handleOpenArtifact(log.artifactId)}
                          role="button"
                          tabIndex={0}
                          onKeyDown={(e) => {
                            if (e.key === "Enter" || e.key === " ") {
                              handleOpenArtifact(log.artifactId);
                            }
                          }}
                        >
                          <span className="log-message">
                            📋 {log.artifactType}: {log.title}
                          </span>
                        </div>
                      );
                    }

                    return (
                      <div key={log.id} className={`log-entry ${log.type}`}>
                        <span className="log-message">{log.message}</span>
                        {log.metadata && (
                          <details className="log-metadata">
                            <summary>Details</summary>
                            <pre>{JSON.stringify(log.metadata, null, 2)}</pre>
                          </details>
                        )}
                      </div>
                    );
                  })}
                  {currentAssistantMessage && (
                    <div className="log-entry assistant streaming">
                      <span className="log-message">{currentAssistantMessage}</span>
                      <span className="streaming-indicator">...</span>
                    </div>
                  )}
                  <div ref={logEndRef} />
                </div>
              </div>

              <div className="message-form">
                <textarea
                  ref={textareaRef}
                  value={userInput}
                  onChange={(e) => setUserInput(e.target.value)}
                  onKeyDown={handleKeyDown}
                  placeholder="Type a message"
                  disabled={!ws.isConnected || isRecording}
                  className="message-input"
                  rows={1}
                />
                <button
                  type="button"
                  onClick={handleToggleRealtimeMode}
                  disabled={!ws.isConnected || isRecording || isVADLoading}
                  className={`send-button realtime-button ${
                    isRealtimeMode ? "active" : ""
                  } ${isSpeechDetected ? "speech-detected" : ""} ${
                    isVADLoading ? "loading" : ""
                  }`}
                  title={
                    isRealtimeMode ? "Stop realtime mode" : "Start realtime mode"
                  }
                >
                  {isVADLoading ? (
                    <span className="loading-indicator" />
                  ) : (
                    <Radio size={20} />
                  )}
                </button>
                <button
                  type="button"
                  onClick={handleButtonClick}
                  disabled={!ws.isConnected}
                  className={`send-button ${isInProgress ? "cancelling" : ""} ${
                    isRecording ? "recording" : ""
                  } ${userInput.trim() ? "has-text" : ""}`}
                >
                  {isInProgress ? (
                    <X size={20} />
                  ) : isRecording ? (
                    <span className="recording-indicator" />
                  ) : userInput.trim() ? (
                    <Send size={20} />
                  ) : (
                    <Mic size={20} />
                  )}
                </button>
              </div>
            </>
          )}

          {activeView === "agent" && activeProcessId && (
            <AgentStreamView
              agentId={activeProcessId}
              agent={agents.get(activeProcessId)!}
              updates={agentUpdates.get(activeProcessId) || []}
              onBack={handleBackToOrchestrator}
              onKillAgent={handleKillAgent}
              onCancelAgent={handleCancelAgent}
            />
          )}

          <ActiveProcesses
            terminals={[]}
            agents={Array.from(agents.values())}
            activeProcessId={activeProcessId}
            activeProcessType={activeView === "orchestrator" ? null : activeView}
            onSelectProcess={handleSelectProcess}
            onBackToOrchestrator={handleBackToOrchestrator}
          />
        </div>
      </main>

      <ArtifactDrawer
        artifact={currentArtifact}
        onClose={handleCloseArtifact}
      />
    </div>
  );
}

export default App;