feat: add Playwright MCP integration and TTS improvements

- Integrate Playwright MCP for browser automation capabilities - Add TTS manager for improved audio synthesis management - Add ToolCallCard UI component for better tool call visualization - Upgrade Vite to v7.1.10 for improved build performance - Add lucide-react for icon components - Add dotenv for environment variable management - Update agent prompt and orchestrator for enhanced interactions - Improve audio playback and voice controls in UI 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-19 23:23:32 +02:00
parent d7a387d911
commit 0bfda11c1f
20 changed files with 1721 additions and 1317 deletions
@@ -0,0 +1,88 @@
+import { v4 as uuidv4 } from "uuid";
+import { synthesizeSpeech } from "./tts-openai.js";
+import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
+
+interface PendingPlayback {
+  resolve: () => void;
+  reject: (error: Error) => void;
+  timeout: NodeJS.Timeout;
+}
+
+/**
+ * Store pending playback confirmations
+ * Maps audio ID -> promise resolve/reject handlers
+ */
+const pendingPlaybacks = new Map<string, PendingPlayback>();
+
+/**
+ * Timeout for waiting on client playback confirmation (30 seconds)
+ */
+const PLAYBACK_TIMEOUT_MS = 30000;
+
+/**
+ * Generate TTS audio, broadcast to clients, and wait for playback confirmation
+ * Returns a Promise that resolves when the client confirms playback completed
+ */
+export async function generateTTSAndWaitForPlayback(
+  text: string,
+  wsServer: VoiceAssistantWebSocketServer
+): Promise<void> {
+  // Generate TTS audio
+  const { audio, format } = await synthesizeSpeech(text);
+
+  // Create unique ID for this audio segment
+  const audioId = uuidv4();
+
+  // Create promise that will be resolved when client confirms playback
+  const playbackPromise = new Promise<void>((resolve, reject) => {
+    // Set timeout to prevent hanging forever
+    const timeout = setTimeout(() => {
+      pendingPlaybacks.delete(audioId);
+      reject(new Error(`Audio playback timeout for ${audioId}`));
+    }, PLAYBACK_TIMEOUT_MS);
+
+    // Store handlers
+    pendingPlaybacks.set(audioId, { resolve, reject, timeout });
+  });
+
+  // Broadcast audio to clients
+  wsServer.broadcast({
+    type: "audio_output",
+    payload: {
+      id: audioId,
+      audio: audio.toString("base64"),
+      format,
+    },
+  });
+
+  console.log(
+    `[TTS-Manager] ${new Date().toISOString()} Sent audio ${audioId}, waiting for playback...`
+  );
+
+  // Wait for playback confirmation
+  await playbackPromise;
+
+  console.log(
+    `[TTS-Manager] ${new Date().toISOString()} Audio ${audioId} playback confirmed`
+  );
+}
+
+/**
+ * Called when client confirms audio playback completed
+ * Resolves the corresponding promise
+ */
+export function confirmAudioPlayed(audioId: string): void {
+  const pending = pendingPlaybacks.get(audioId);
+
+  if (!pending) {
+    console.warn(
+      `[TTS-Manager] Received confirmation for unknown audio ID: ${audioId}`
+    );
+    return;
+  }
+
+  // Clear timeout and resolve promise
+  clearTimeout(pending.timeout);
+  pending.resolve();
+  pendingPlaybacks.delete(audioId);
+}