feat: add realtime voice mode, artifact presentation, and Playwright MCP integration

packages/voice-assistant/src/server/agent/llm-openai.ts

@@ -11,6 +11,7 @@ import {
   killTerminal,
 } from "../daemon/terminal-manager.js";
 import invariant from "tiny-invariant";
+import { getPlaywrightTools } from "./playwright-mcp.js";
 
 /**
  * Terminal tools using Vercel AI SDK tool() function
@@ -208,8 +209,68 @@ export const terminalTools = {
       return { success: true };
     },
   }),
+
+  present_artifact: tool({
+    description:
+      "Present an artifact (plan, diff, screenshot, etc.) to the user for review. Use this when you need to show information that's hard to convey via TTS, such as markdown plans, code diffs, or visual content",
+    inputSchema: z.object({
+      type: z
+        .enum(["markdown", "diff", "image", "code"])
+        .describe("Type of artifact to present."),
+      source: z.union([
+        z.object({
+          type: z.literal("file"),
+          path: z.string(),
+        }),
+        z.object({
+          type: z.literal("command_output"),
+          command: z.string(),
+        }),
+        z.object({
+          type: z.literal("text"),
+          text: z.string(),
+        }),
+      ]),
+      title: z
+        .string()
+        .describe(
+          "Title for the artifact (e.g., 'Implementation Plan', 'Refactoring Strategy', '/path/to/project/package.json')."
+        ),
+    }),
+    execute: async () => {
+      // Artifact will be broadcast by orchestrator via onToolCall callback
+      // We just return a simple acknowledgment here
+      return {
+        success: true,
+        message: "Artifact presented to user.",
+      };
+    },
+  }),
 };
 
+/**
+ * Cache Playwright MCP tools (lazy loaded on first use)
+ */
+let playwrightToolsCache: Record<string, any> | null = null;
+let playwrightToolsPromise: Promise<Record<string, any>> | null = null;
+
+async function getCachedPlaywrightTools(): Promise<Record<string, any>> {
+  if (playwrightToolsCache) {
+    return playwrightToolsCache;
+  }
+
+  if (playwrightToolsPromise) {
+    return playwrightToolsPromise;
+  }
+
+  playwrightToolsPromise = getPlaywrightTools().then((tools) => {
+    playwrightToolsCache = tools;
+    return tools;
+  });
+
+  return playwrightToolsPromise;
+}
+
 /**
  * Message interface for conversation
  */
@@ -252,11 +313,18 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
     apiKey: process.env.OPENROUTER_API_KEY,
   });
 
+  // Merge terminal tools with Playwright MCP tools
+  const pwTools = await getCachedPlaywrightTools();
+  const allTools = {
+    ...terminalTools,
+    ...pwTools,
+  };
+
   const result = await streamText({
     model: openrouter("anthropic/claude-haiku-4.5"),
     system: params.systemPrompt,
     messages: params.messages,
-    tools: terminalTools,
+    tools: allTools,
     abortSignal: params.abortSignal,
     onChunk: async ({ chunk }) => {
       // console.log("onChunk", chunk);

packages/voice-assistant/src/server/agent/orchestrator.ts

@@ -1,8 +1,14 @@
 import { v4 as uuidv4 } from "uuid";
+import { readFile } from "fs/promises";
+import { exec } from "child_process";
+import { promisify } from "util";
 import { getSystemPrompt } from "./system-prompt.js";
 import { streamLLM, type Message } from "./llm-openai.js";
 import { generateTTSAndWaitForPlayback } from "./tts-manager.js";
 import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
+import type { ArtifactPayload } from "../types.js";
+
+const execAsync = promisify(exec);
 
 interface ConversationContext {
   id: string;
@@ -109,6 +115,60 @@ export async function processUserMessage(params: {
           await pendingTTS;
           pendingTTS = null;
         }
+
+        // Handle present_artifact tool specially
+        if (toolName === "present_artifact" && params.wsServer) {
+          const artifactId = uuidv4();
+
+          // Resolve source to content
+          let content: string;
+          let isBase64 = false;
+          try {
+            if (args.source.type === "file") {
+              const fileBuffer = await readFile(args.source.path);
+              content = fileBuffer.toString("base64");
+              isBase64 = true;
+            } else if (args.source.type === "command_output") {
+              const { stdout } = await execAsync(args.source.command, { encoding: 'buffer' });
+              content = stdout.toString("base64");
+              isBase64 = true;
+            } else if (args.source.type === "text") {
+              content = args.source.text;
+              isBase64 = false;
+            } else {
+              content = "[Unknown source type]";
+              isBase64 = false;
+            }
+          } catch (error) {
+            console.error("Failed to resolve artifact source:", error);
+            content = `[Error resolving source: ${error instanceof Error ? error.message : String(error)}]`;
+            isBase64 = false;
+          }
+
+          const artifact: ArtifactPayload = {
+            type: args.type,
+            id: artifactId,
+            title: args.title,
+            content,
+            isBase64,
+          };
+
+          // Broadcast artifact to client
+          params.wsServer.broadcast({
+            type: "artifact",
+            payload: artifact,
+          });
+
+          // Broadcast as activity log entry so it appears in the feed
+          params.wsServer.broadcastActivityLog({
+            id: artifactId,
+            timestamp: new Date(),
+            type: "system",
+            content: `${args.type} artifact: ${args.title}`,
+            metadata: { artifactId, artifactType: args.type },
+          });
+        }
+
         // Broadcast tool call to WebSocket
         if (params.wsServer) {
           params.wsServer.broadcastActivityLog({
@@ -240,3 +300,4 @@ export function getConversationStats(): {
     })),
   };
 }
+

packages/voice-assistant/src/server/agent/playwright-mcp.ts

@@ -0,0 +1,55 @@
+import { experimental_createMCPClient } from "ai";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+
+let playwrightMCPClient: Awaited<
+  ReturnType<typeof experimental_createMCPClient>
+> | null = null;
+
+/**
+ * Get or create the Playwright MCP client (singleton)
+ */
+export async function getPlaywrightMCPClient() {
+  if (playwrightMCPClient) {
+    return playwrightMCPClient;
+  }
+
+  const transport = new StdioClientTransport({
+    command: "npx",
+    args: ["@playwright/mcp", "--browser", "firefox"],
+  });
+
+  playwrightMCPClient = await experimental_createMCPClient({
+    transport,
+  });
+
+  console.log("Playwright MCP client initialized with Firefox");
+
+  return playwrightMCPClient;
+}
+
+/**
+ * Get Playwright MCP tools in AI SDK format
+ */
+export async function getPlaywrightTools() {
+  try {
+    const client = await getPlaywrightMCPClient();
+    const tools = await client.tools();
+    console.log(`Loaded ${Object.keys(tools).length} Playwright MCP tools`);
+    return tools;
+  } catch (error) {
+    console.error("Failed to initialize Playwright MCP tools:", error);
+    // Return empty object to allow app to continue without Playwright
+    return {};
+  }
+}
+
+/**
+ * Cleanup function to close the MCP client
+ */
+export async function closePlaywrightMCP() {
+  if (playwrightMCPClient) {
+    await playwrightMCPClient.close();
+    playwrightMCPClient = null;
+    console.log("Playwright MCP client closed");
+  }
+}