Based on the git diff showing extensive package-lock.json changes and modifications to the agent system prompt and configuration, here's the commit message:

```
feat: add AI SDK dependencies and refine agent voice interaction config

- Add @ai-sdk packages for enhanced AI capabilities
- Install Deepgram SDK, livekit-client, and werift for voice/WebRTC
- Expand agent system prompt with multi-tool workflows and Claude Code interaction patterns
- Adjust VAD timing (1.2s silence threshold) and turn detection for better voice UX
- Switch TTS to inworld/inworld-tts-1 with Olivia voice
- Configure interruption timing (0.2s detection, no word requirement)
- Add comprehensive guidelines for conversational responses vs structured output
- Document plan mode triggers and Claude response type handling

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
```

packages/voice-assistant/src/server/agent/llm-openai.ts

@@ -0,0 +1,173 @@
+import { streamText, tool } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+import { z } from 'zod';
+import {
+  listTerminals,
+  createTerminal,
+  captureTerminal,
+  sendText,
+  renameTerminal,
+  killTerminal,
+} from '../daemon/terminal-manager.js';
+
+let openaiClient: ReturnType<typeof createOpenAI> | null = null;
+
+/**
+ * Initialize OpenAI client with API key
+ */
+export function initializeOpenAI(apiKey: string): void {
+  openaiClient = createOpenAI({ apiKey });
+}
+
+/**
+ * Terminal tools using Vercel AI SDK tool() function
+ * These are automatically executed when the LLM calls them
+ */
+export const terminalTools = {
+  list_terminals: tool({
+    description: 'List all available terminals in the voice-dev session',
+    parameters: z.object({}),
+    execute: async () => {
+      const terminals = await listTerminals();
+      return { terminals };
+    },
+  }),
+
+  create_terminal: tool({
+    description: 'Create a new terminal with specified name and working directory',
+    parameters: z.object({
+      name: z.string().describe('Unique name for the terminal'),
+      workingDirectory: z.string().describe('Working directory path'),
+      initialCommand: z.string().optional().describe('Optional command to run on creation'),
+    }),
+    execute: async ({ name, workingDirectory, initialCommand }) => {
+      const terminal = await createTerminal({ name, workingDirectory, initialCommand });
+      return { terminal };
+    },
+  }),
+
+  capture_terminal: tool({
+    description: 'Capture and return the output from a terminal',
+    parameters: z.object({
+      terminalId: z.string().describe('Terminal ID (e.g., @123)'),
+      lines: z.number().optional().describe('Number of lines to capture (default: 200)'),
+      wait: z.number().optional().describe('Milliseconds to wait before capture'),
+    }),
+    execute: async ({ terminalId, lines, wait }) => {
+      const output = await captureTerminal(terminalId, lines, wait);
+      return { output };
+    },
+  }),
+
+  send_text: tool({
+    description: 'Send text or commands to a terminal',
+    parameters: z.object({
+      terminalId: z.string().describe('Terminal ID (e.g., @123)'),
+      text: z.string().describe('Text to send to the terminal'),
+      pressEnter: z.boolean().optional().describe('Whether to press Enter after text'),
+      return_output: z
+        .object({
+          lines: z.number().optional(),
+          wait: z.number().optional(),
+        })
+        .optional()
+        .describe('Capture output after sending'),
+    }),
+    execute: async ({ terminalId, text, pressEnter, return_output }) => {
+      const output = await sendText(terminalId, text, pressEnter, return_output);
+      return { output };
+    },
+  }),
+
+  rename_terminal: tool({
+    description: 'Rename an existing terminal',
+    parameters: z.object({
+      terminalId: z.string().describe('Terminal ID to rename'),
+      newName: z.string().describe('New unique name for the terminal'),
+    }),
+    execute: async ({ terminalId, newName }) => {
+      await renameTerminal(terminalId, newName);
+      return { success: true };
+    },
+  }),
+
+  kill_terminal: tool({
+    description: 'Close and destroy a terminal',
+    parameters: z.object({
+      terminalId: z.string().describe('Terminal ID to kill'),
+    }),
+    execute: async ({ terminalId }) => {
+      await killTerminal(terminalId);
+      return { success: true };
+    },
+  }),
+};
+
+/**
+ * Message interface for conversation
+ */
+export interface Message {
+  role: 'system' | 'user' | 'assistant';
+  content: string;
+}
+
+/**
+ * Streaming LLM parameters
+ */
+export interface StreamLLMParams {
+  messages: Message[];
+  onChunk?: (chunk: string) => void;
+  onToolCall?: (toolName: string, args: any) => void;
+  onToolResult?: (toolName: string, result: any) => void;
+  onFinish?: (fullText: string) => void;
+}
+
+/**
+ * Stream LLM response with automatic tool execution
+ */
+export async function streamLLM(params: StreamLLMParams): Promise<string> {
+  if (!openaiClient) {
+    throw new Error('OpenAI client not initialized. Call initializeOpenAI() first.');
+  }
+
+  const result = await streamText({
+    model: openaiClient('gpt-4o') as any, // Type assertion to work around AI SDK version conflicts
+    messages: params.messages,
+    tools: terminalTools,
+    maxSteps: 5, // Allow up to 5 tool executions per turn
+    onStepFinish: async (event) => {
+      // Called after each step (tool call or text generation)
+      if (event.toolCalls && event.toolCalls.length > 0) {
+        for (const toolCall of event.toolCalls) {
+          if (params.onToolCall) {
+            params.onToolCall(toolCall.toolName, toolCall.args);
+          }
+        }
+      }
+
+      if (event.toolResults && event.toolResults.length > 0) {
+        for (const toolResult of event.toolResults) {
+          if (params.onToolResult) {
+            params.onToolResult(toolResult.toolName, toolResult.result);
+          }
+        }
+      }
+    },
+  });
+
+  let fullText = '';
+
+  // Stream text chunks to the caller
+  for await (const chunk of result.textStream) {
+    fullText += chunk;
+    if (params.onChunk) {
+      params.onChunk(chunk);
+    }
+  }
+
+  if (params.onFinish) {
+    params.onFinish(fullText);
+  }
+
+  return fullText;
+}

packages/voice-assistant/src/server/agent/orchestrator.ts

@@ -0,0 +1,173 @@
+import { v4 as uuidv4 } from 'uuid';
+import { getSystemPrompt } from './system-prompt.js';
+import { streamLLM, type Message } from './llm-openai.js';
+import type { VoiceAssistantWebSocketServer } from '../websocket-server.js';
+
+interface ConversationContext {
+  id: string;
+  messages: Message[];
+  createdAt: Date;
+  lastActivity: Date;
+}
+
+/**
+ * Store active conversations (in-memory for now)
+ * In production, this could be persisted to a database
+ */
+const conversations = new Map<string, ConversationContext>();
+
+/**
+ * Create a new conversation with system prompt
+ */
+export function createConversation(): string {
+  const id = uuidv4();
+  conversations.set(id, {
+    id,
+    messages: [{ role: 'system', content: getSystemPrompt() }],
+    createdAt: new Date(),
+    lastActivity: new Date(),
+  });
+  return id;
+}
+
+/**
+ * Get conversation by ID
+ */
+export function getConversation(id: string): ConversationContext | null {
+  return conversations.get(id) || null;
+}
+
+/**
+ * Process user message through the LLM orchestrator
+ * Handles streaming, tool calls, and WebSocket broadcasting
+ */
+export async function processUserMessage(params: {
+  conversationId: string;
+  message: string;
+  wsServer?: VoiceAssistantWebSocketServer;
+  enableTTS?: boolean;
+}): Promise<string> {
+  const conversation = conversations.get(params.conversationId);
+  if (!conversation) {
+    throw new Error('Conversation not found');
+  }
+
+  // Add user message to context
+  conversation.messages.push({
+    role: 'user',
+    content: params.message,
+  });
+  conversation.lastActivity = new Date();
+
+  // Broadcast user message to WebSocket clients
+  if (params.wsServer) {
+    params.wsServer.broadcastActivityLog({
+      id: uuidv4(),
+      timestamp: new Date(),
+      type: 'transcript',
+      content: params.message,
+    });
+  }
+
+  let assistantResponse = '';
+
+  try {
+    // Stream LLM response with tool execution
+    assistantResponse = await streamLLM({
+      messages: conversation.messages,
+      onChunk: (chunk) => {
+        // Broadcast streaming chunks to WebSocket
+        if (params.wsServer) {
+          params.wsServer.broadcast({
+            type: 'assistant_chunk',
+            payload: { chunk },
+          });
+        }
+      },
+      onToolCall: (toolName, args) => {
+        // Broadcast tool call to WebSocket
+        if (params.wsServer) {
+          params.wsServer.broadcastActivityLog({
+            id: uuidv4(),
+            timestamp: new Date(),
+            type: 'tool_call',
+            content: `Calling ${toolName}`,
+            metadata: { toolName, arguments: args },
+          });
+        }
+      },
+      onToolResult: (toolName, result) => {
+        // Broadcast tool result to WebSocket
+        if (params.wsServer) {
+          params.wsServer.broadcastActivityLog({
+            id: uuidv4(),
+            timestamp: new Date(),
+            type: 'tool_result',
+            content: `Tool ${toolName} completed`,
+            metadata: { toolName, result },
+          });
+        }
+      },
+      onFinish: (fullText) => {
+        // Broadcast complete assistant response
+        if (params.wsServer) {
+          params.wsServer.broadcastActivityLog({
+            id: uuidv4(),
+            timestamp: new Date(),
+            type: 'assistant',
+            content: fullText,
+          });
+        }
+      },
+    });
+
+    // Add assistant response to context
+    conversation.messages.push({
+      role: 'assistant',
+      content: assistantResponse,
+    });
+  } catch (error: any) {
+    // Broadcast error to WebSocket
+    if (params.wsServer) {
+      params.wsServer.broadcastActivityLog({
+        id: uuidv4(),
+        timestamp: new Date(),
+        type: 'error',
+        content: `Error: ${error.message}`,
+      });
+    }
+    throw error;
+  }
+
+  return assistantResponse;
+}
+
+/**
+ * Clean up old conversations (call periodically)
+ */
+export function cleanupConversations(maxAgeMinutes: number = 60): void {
+  const now = new Date();
+  for (const [id, conv] of conversations.entries()) {
+    const ageMinutes = (now.getTime() - conv.lastActivity.getTime()) / (1000 * 60);
+    if (ageMinutes > maxAgeMinutes) {
+      conversations.delete(id);
+    }
+  }
+}
+
+/**
+ * Get conversation statistics
+ */
+export function getConversationStats(): {
+  total: number;
+  conversations: Array<{ id: string; messageCount: number; lastActivity: Date }>;
+} {
+  return {
+    total: conversations.size,
+    conversations: Array.from(conversations.values()).map((conv) => ({
+      id: conv.id,
+      messageCount: conv.messages.length,
+      lastActivity: conv.lastActivity,
+    })),
+  };
+}

packages/voice-assistant/src/server/agent/stt-openai.ts

@@ -0,0 +1,100 @@
+import OpenAI from 'openai';
+import { writeFile, unlink } from 'fs/promises';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { v4 as uuidv4 } from 'uuid';
+
+export interface STTConfig {
+  apiKey: string;
+  model?: 'whisper-1';
+}
+
+export interface TranscriptionResult {
+  text: string;
+  language?: string;
+  duration?: number;
+}
+
+let openaiClient: OpenAI | null = null;
+let config: STTConfig | null = null;
+
+export function initializeSTT(sttConfig: STTConfig): void {
+  config = sttConfig;
+  openaiClient = new OpenAI({
+    apiKey: sttConfig.apiKey,
+  });
+  console.log('✓ STT (OpenAI Whisper) initialized');
+}
+
+export async function transcribeAudio(
+  audioBuffer: Buffer,
+  format: string
+): Promise<TranscriptionResult> {
+  if (!openaiClient || !config) {
+    throw new Error('STT not initialized. Call initializeSTT() first.');
+  }
+
+  const startTime = Date.now();
+  let tempFilePath: string | null = null;
+
+  try {
+    // Map format to file extension
+    const ext = getFileExtension(format);
+
+    // Write audio buffer to temporary file
+    // OpenAI API requires file upload, not raw buffer
+    tempFilePath = join(tmpdir(), `audio-${uuidv4()}.${ext}`);
+    await writeFile(tempFilePath, audioBuffer);
+
+    console.log(`[STT] Transcribing audio file: ${tempFilePath} (${audioBuffer.length} bytes)`);
+
+    // Call OpenAI Whisper API
+    const response = await openaiClient.audio.transcriptions.create({
+      file: await import('fs').then((fs) => fs.createReadStream(tempFilePath!)),
+      model: config.model || 'whisper-1',
+      response_format: 'verbose_json', // Get language and duration info
+    });
+
+    const duration = Date.now() - startTime;
+
+    console.log(`[STT] Transcription complete in ${duration}ms: "${response.text}"`);
+
+    return {
+      text: response.text,
+      language: response.language,
+      duration: duration,
+    };
+  } catch (error: any) {
+    console.error('[STT] Transcription error:', error);
+    throw new Error(`STT transcription failed: ${error.message}`);
+  } finally {
+    // Clean up temporary file
+    if (tempFilePath) {
+      try {
+        await unlink(tempFilePath);
+      } catch (cleanupError) {
+        console.warn(`[STT] Failed to clean up temp file: ${tempFilePath}`);
+      }
+    }
+  }
+}
+
+function getFileExtension(format: string): string {
+  // Map mime types or format strings to file extensions
+  const formatLower = format.toLowerCase();
+
+  if (formatLower.includes('webm')) return 'webm';
+  if (formatLower.includes('ogg')) return 'ogg';
+  if (formatLower.includes('mp3')) return 'mp3';
+  if (formatLower.includes('wav')) return 'wav';
+  if (formatLower.includes('m4a')) return 'm4a';
+  if (formatLower.includes('mp4')) return 'mp4';
+  if (formatLower.includes('flac')) return 'flac';
+
+  // Default to webm (common browser format)
+  return 'webm';
+}
+
+export function isSTTInitialized(): boolean {
+  return openaiClient !== null && config !== null;
+}

packages/voice-assistant/src/server/agent/system-prompt.ts

@@ -0,0 +1,22 @@
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import path from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Load system prompt from agent-prompt.md in package root
+export function loadSystemPrompt(): string {
+  const promptPath = path.join(__dirname, '../../../agent-prompt.md');
+  return readFileSync(promptPath, 'utf-8');
+}
+
+// Cache the loaded prompt
+let cachedPrompt: string | null = null;
+
+export function getSystemPrompt(): string {
+  if (!cachedPrompt) {
+    cachedPrompt = loadSystemPrompt();
+  }
+  return cachedPrompt;
+}

packages/voice-assistant/src/server/agent/tts-openai.ts

@@ -0,0 +1,94 @@
+import OpenAI from 'openai';
+import { Readable } from 'stream';
+
+export interface TTSConfig {
+  apiKey: string;
+  model?: 'tts-1' | 'tts-1-hd';
+  voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
+  responseFormat?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm';
+}
+
+export interface SpeechResult {
+  audio: Buffer;
+  format: string;
+}
+
+let openaiClient: OpenAI | null = null;
+let config: TTSConfig | null = null;
+
+export function initializeTTS(ttsConfig: TTSConfig): void {
+  config = {
+    model: 'tts-1',
+    voice: 'alloy',
+    responseFormat: 'mp3',
+    ...ttsConfig,
+  };
+  openaiClient = new OpenAI({
+    apiKey: ttsConfig.apiKey,
+  });
+  console.log(`✓ TTS (OpenAI) initialized with voice: ${config.voice}, model: ${config.model}`);
+}
+
+export async function synthesizeSpeech(text: string): Promise<SpeechResult> {
+  if (!openaiClient || !config) {
+    throw new Error('TTS not initialized. Call initializeTTS() first.');
+  }
+
+  if (!text || text.trim().length === 0) {
+    throw new Error('Cannot synthesize empty text');
+  }
+
+  const startTime = Date.now();
+
+  try {
+    console.log(`[TTS] Synthesizing speech: "${text.substring(0, 50)}${text.length > 50 ? '...' : ''}"`);
+
+    // Call OpenAI TTS API with streaming
+    const response = await openaiClient.audio.speech.create({
+      model: config.model!,
+      voice: config.voice!,
+      input: text,
+      response_format: config.responseFormat as 'mp3' | 'opus' | 'aac' | 'flac',
+    });
+
+    // Response is a stream, convert to buffer
+    const audioBuffer = await streamToBuffer(response.body as unknown as Readable);
+
+    const duration = Date.now() - startTime;
+    console.log(`[TTS] Speech synthesis complete in ${duration}ms (${audioBuffer.length} bytes)`);
+
+    return {
+      audio: audioBuffer,
+      format: config.responseFormat || 'mp3',
+    };
+  } catch (error: any) {
+    console.error('[TTS] Speech synthesis error:', error);
+    throw new Error(`TTS synthesis failed: ${error.message}`);
+  }
+}
+
+async function streamToBuffer(stream: Readable): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+
+  return new Promise((resolve, reject) => {
+    stream.on('data', (chunk) => {
+      chunks.push(Buffer.from(chunk));
+    });
+
+    stream.on('end', () => {
+      resolve(Buffer.concat(chunks));
+    });
+
+    stream.on('error', (error) => {
+      reject(error);
+    });
+  });
+}
+
+export function isTTSInitialized(): boolean {
+  return openaiClient !== null && config !== null;
+}
+
+export function getTTSConfig(): TTSConfig | null {
+  return config;
+}

packages/voice-assistant/src/server/daemon/terminal-manager.ts

@@ -0,0 +1,201 @@
+import {
+  findSessionByName,
+  createSession,
+  listWindows,
+  createWindow,
+  capturePaneContent,
+  sendText as tmuxSendText,
+  renameWindow,
+  killWindow,
+  isWindowNameUnique,
+  getCurrentWorkingDirectory,
+  getCurrentCommand,
+} from "./tmux.js";
+
+const DEFAULT_SESSION = "voice-dev";
+
+// Terminal model: session → windows (single pane per window)
+// Terminal ID = window ID (format: @123)
+
+export interface TerminalInfo {
+  id: string; // window ID
+  name: string;
+  workingDirectory: string;
+  currentCommand: string;
+}
+
+export interface CreateTerminalParams {
+  name: string;
+  workingDirectory: string;
+  initialCommand?: string;
+}
+
+export interface Terminal extends TerminalInfo {
+  active: boolean;
+  sessionId: string;
+}
+
+/**
+ * Initialize the default "voice-dev" tmux session
+ * Creates it if it doesn't exist
+ */
+export async function initializeDefaultSession(): Promise<void> {
+  const session = await findSessionByName(DEFAULT_SESSION);
+
+  if (!session) {
+    await createSession(DEFAULT_SESSION);
+  }
+}
+
+/**
+ * List all terminals in the voice-dev session
+ * Returns terminal info including ID, name, working directory, and current command
+ */
+export async function listTerminals(): Promise<TerminalInfo[]> {
+  const session = await findSessionByName(DEFAULT_SESSION);
+
+  if (!session) {
+    throw new Error(`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`);
+  }
+
+  const windows = await listWindows(session.id);
+
+  const terminals: TerminalInfo[] = [];
+
+  for (const window of windows) {
+    // Get the first (and only) pane in this window
+    const paneId = `${window.id}.0`;
+
+    try {
+      const workingDirectory = await getCurrentWorkingDirectory(paneId);
+      const currentCommand = await getCurrentCommand(paneId);
+
+      terminals.push({
+        id: window.id,
+        name: window.name,
+        workingDirectory,
+        currentCommand,
+      });
+    } catch (error) {
+      // If we can't get pane info, still include the terminal with empty values
+      terminals.push({
+        id: window.id,
+        name: window.name,
+        workingDirectory: "",
+        currentCommand: "",
+      });
+    }
+  }
+
+  return terminals;
+}
+
+/**
+ * Create a new terminal (tmux window) with specified name and working directory
+ * Optionally execute an initial command
+ */
+export async function createTerminal(params: CreateTerminalParams): Promise<Terminal> {
+  const session = await findSessionByName(DEFAULT_SESSION);
+
+  if (!session) {
+    throw new Error(`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`);
+  }
+
+  // Validate name uniqueness
+  const isUnique = await isWindowNameUnique(session.id, params.name);
+  if (!isUnique) {
+    throw new Error(
+      `Terminal with name '${params.name}' already exists. Please choose a unique name.`
+    );
+  }
+
+  // Create the window
+  const windowResult = await createWindow(session.id, params.name, {
+    workingDirectory: params.workingDirectory,
+    command: params.initialCommand,
+  });
+
+  if (!windowResult) {
+    throw new Error(`Failed to create terminal '${params.name}'`);
+  }
+
+  const paneId = windowResult.paneId;
+
+  // Get terminal info
+  const workingDirectory = await getCurrentWorkingDirectory(paneId);
+  const currentCommand = await getCurrentCommand(paneId);
+
+  return {
+    id: windowResult.id,
+    name: windowResult.name,
+    active: windowResult.active,
+    sessionId: session.id,
+    workingDirectory,
+    currentCommand,
+  };
+}
+
+/**
+ * Capture output from a terminal
+ * Returns the last N lines of terminal content
+ */
+export async function captureTerminal(
+  terminalId: string,
+  lines: number = 200,
+  wait?: number
+): Promise<string> {
+  // Terminal ID is window ID, get the first pane
+  const paneId = `${terminalId}.0`;
+
+  // Optional wait before capture
+  if (wait) {
+    await new Promise((resolve) => setTimeout(resolve, wait));
+  }
+
+  return capturePaneContent(paneId, lines, false);
+}
+
+/**
+ * Send text to a terminal, optionally press Enter, optionally return output
+ */
+export async function sendText(
+  terminalId: string,
+  text: string,
+  pressEnter: boolean = false,
+  return_output?: { lines?: number; wait?: number }
+): Promise<string | void> {
+  // Terminal ID is window ID, get the first pane
+  const paneId = `${terminalId}.0`;
+
+  return tmuxSendText({
+    paneId,
+    text,
+    pressEnter,
+    return_output,
+  });
+}
+
+/**
+ * Rename a terminal
+ * Validates that the new name is unique
+ */
+export async function renameTerminal(
+  terminalId: string,
+  newName: string
+): Promise<void> {
+  const session = await findSessionByName(DEFAULT_SESSION);
+
+  if (!session) {
+    throw new Error(`Session '${DEFAULT_SESSION}' not found.`);
+  }
+
+  // renameWindow handles uniqueness validation internally
+  await renameWindow(session.id, terminalId, newName);
+}
+
+/**
+ * Kill (close/destroy) a terminal
+ */
+export async function killTerminal(terminalId: string): Promise<void> {
+  await killWindow(terminalId);
+}

packages/voice-assistant/src/server/daemon/tmux.ts

@@ -0,0 +1,852 @@
+import { exec as execCallback } from "child_process";
+import { promisify } from "util";
+import { v4 as uuidv4 } from "uuid";
+import os from "node:os";
+
+const exec = promisify(execCallback);
+
+// Basic interfaces for tmux objects
+export interface TmuxSession {
+  id: string;
+  name: string;
+  attached: boolean;
+  windows: number;
+}
+
+export interface TmuxWindow {
+  id: string;
+  name: string;
+  active: boolean;
+  sessionId: string;
+}
+
+export interface TmuxPane {
+  id: string;
+  windowId: string;
+  active: boolean;
+  title: string;
+}
+
+interface CommandExecution {
+  id: string;
+  paneId: string;
+  command: string;
+  status: "pending" | "completed" | "error";
+  startTime: Date;
+  result?: string;
+  exitCode?: number;
+  rawMode?: boolean;
+}
+
+export type ShellType = "bash" | "zsh" | "fish";
+
+let shellConfig: { type: ShellType } = { type: "bash" };
+
+export function setShellConfig(config: { type: string }): void {
+  // Validate shell type
+  const validShells: ShellType[] = ["bash", "zsh", "fish"];
+
+  if (validShells.includes(config.type as ShellType)) {
+    shellConfig = { type: config.type as ShellType };
+  } else {
+    shellConfig = { type: "bash" };
+  }
+}
+
+/**
+ * Execute a tmux command and return the result
+ */
+export async function executeTmux(tmuxCommand: string): Promise<string> {
+  try {
+    const { stdout } = await exec(`tmux ${tmuxCommand}`);
+    return stdout.trim();
+  } catch (error: any) {
+    throw new Error(`Failed to execute tmux command: ${error.message}`);
+  }
+}
+
+/**
+ * Check if tmux server is running
+ */
+export async function isTmuxRunning(): Promise<boolean> {
+  try {
+    await executeTmux("list-sessions -F '#{session_name}'");
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+
+/**
+ * List all tmux sessions
+ */
+export async function listSessions(): Promise<TmuxSession[]> {
+  const format =
+    "#{session_id}:#{session_name}:#{?session_attached,1,0}:#{session_windows}";
+  const output = await executeTmux(`list-sessions -F '${format}'`);
+
+  if (!output) return [];
+
+  return output.split("\n").map((line) => {
+    const [id, name, attached, windows] = line.split(":");
+    return {
+      id,
+      name,
+      attached: attached === "1",
+      windows: parseInt(windows, 10),
+    };
+  });
+}
+
+/**
+ * Find a session by name
+ */
+export async function findSessionByName(
+  name: string
+): Promise<TmuxSession | null> {
+  try {
+    const sessions = await listSessions();
+    return sessions.find((session) => session.name === name) || null;
+  } catch (error) {
+    return null;
+  }
+}
+
+/**
+ * Find a window by name in a session
+ */
+export async function findWindowByName(
+  sessionId: string,
+  name: string
+): Promise<TmuxWindow | null> {
+  try {
+    const windows = await listWindows(sessionId);
+    return windows.find((window) => window.name === name) || null;
+  } catch (error) {
+    return null;
+  }
+}
+
+/**
+ * Check if a window name is unique in a session
+ */
+export async function isWindowNameUnique(
+  sessionId: string,
+  name: string
+): Promise<boolean> {
+  const window = await findWindowByName(sessionId, name);
+  return window === null;
+}
+
+/**
+ * List windows in a session
+ */
+export async function listWindows(sessionId: string): Promise<TmuxWindow[]> {
+  const format = "#{window_id}:#{window_name}:#{?window_active,1,0}";
+  const output = await executeTmux(
+    `list-windows -t '${sessionId}' -F '${format}'`
+  );
+
+  if (!output) return [];
+
+  return output.split("\n").map((line) => {
+    const [id, name, active] = line.split(":");
+    return {
+      id,
+      name,
+      active: active === "1",
+      sessionId,
+    };
+  });
+}
+
+/**
+ * List panes in a window
+ */
+export async function listPanes(windowId: string): Promise<TmuxPane[]> {
+  const format = "#{pane_id}:#{pane_title}:#{?pane_active,1,0}";
+  const output = await executeTmux(
+    `list-panes -t '${windowId}' -F '${format}'`
+  );
+
+  if (!output) return [];
+
+  return output.split("\n").map((line) => {
+    const [id, title, active] = line.split(":");
+    return {
+      id,
+      windowId,
+      title: title,
+      active: active === "1",
+    };
+  });
+}
+
+/**
+ * Capture content from a specific pane, by default the latest 200 lines.
+ */
+export async function capturePaneContent(
+  paneId: string,
+  lines: number = 200,
+  includeColors: boolean = false
+): Promise<string> {
+  const colorFlag = includeColors ? "-e" : "";
+  return executeTmux(
+    `capture-pane -p ${colorFlag} -t '${paneId}' -S -${lines} -E -`
+  );
+}
+
+/**
+ * Get the current working directory of a pane
+ */
+export async function getCurrentWorkingDirectory(
+  paneId: string
+): Promise<string> {
+  try {
+    const tmuxPath = await executeTmux(`display-message -p -t '${paneId}' '#{pane_current_path}'`);
+
+    // If tmux returns a valid path, use it
+    if (tmuxPath && tmuxPath.trim()) {
+      return tmuxPath;
+    }
+
+    // Fallback: get the PID and use lsof to find the actual CWD
+    const shellPid = await executeTmux(`display-message -p -t '${paneId}' '#{pane_pid}'`);
+    const { stdout } = await exec(`lsof -a -p ${shellPid.trim()} -d cwd -Fn | grep '^n' | cut -c2-`);
+    return stdout.trim() || tmuxPath;
+  } catch (error) {
+    // If all else fails, return empty string
+    return "";
+  }
+}
+
+/**
+ * Get the current command running in a pane (full command line with arguments)
+ * Gets the immediate child process of the shell, not the shell itself
+ */
+export async function getCurrentCommand(paneId: string): Promise<string> {
+  try {
+    // Get the shell PID (the pane's main process)
+    const shellPid = await executeTmux(`display-message -p -t '${paneId}' '#{pane_pid}'`);
+
+    // First, check if there's a child process using comm= (works for all programs including top)
+    // Use 'ax' flags to see all processes
+    const { stdout: childPid } = await exec(`ps ax -o pid=,ppid=,comm= | awk '$2 == ${shellPid.trim()} { print $1; exit }'`);
+
+    if (childPid.trim()) {
+      // Found a child process, get its full command with args
+      const { stdout: fullCmd } = await exec(`ps -p ${childPid.trim()} -o args= | sed 's/\\\\012.*//'`);
+      const command = fullCmd.trim();
+      if (command) {
+        return command;
+      }
+    }
+
+    // No child process, just return the shell name
+    const { stdout: shellCmd } = await exec(`ps -p ${shellPid} -o comm=`);
+    return shellCmd.trim();
+  } catch (error) {
+    // Fallback to just the command name if ps fails
+    return executeTmux(`display-message -p -t '${paneId}' '#{pane_current_command}'`);
+  }
+}
+
+/**
+ * Create a new tmux session with a default window named "default" in home directory
+ */
+export async function createSession(name: string): Promise<TmuxSession | null> {
+  const homeDir = process.env.HOME || "~";
+  await executeTmux(`new-session -d -s "${name}" -n "default" -c "${homeDir}"`);
+
+  // Disable automatic window renaming for all windows in the session
+  await executeTmux(`set-window-option -t "${name}" automatic-rename off`);
+
+  return findSessionByName(name);
+}
+
+/**
+ * Expand tilde in path to home directory
+ */
+function expandTilde(path: string): string {
+  if (path.startsWith('~/')) {
+    const homeDir = process.env.HOME || os.homedir();
+    return path.replace('~', homeDir);
+  }
+  if (path === '~') {
+    return process.env.HOME || os.homedir();
+  }
+  return path;
+}
+
+/**
+ * Create a new window in a session with optional working directory and initial command
+ */
+export async function createWindow(
+  sessionId: string,
+  name: string,
+  options?: {
+    workingDirectory?: string;
+    command?: string;
+  }
+): Promise<(TmuxWindow & { paneId: string; output?: string }) | null> {
+  // Validate name uniqueness
+  const isUnique = await isWindowNameUnique(sessionId, name);
+  if (!isUnique) {
+    throw new Error(
+      `Terminal with name '${name}' already exists. Please choose a unique name.`
+    );
+  }
+
+  // Build new-window command with optional working directory
+  let newWindowCmd = `new-window -t '${sessionId}' -n '${name}'`;
+  if (options?.workingDirectory) {
+    // Expand tilde to home directory before passing to tmux
+    const expandedPath = expandTilde(options.workingDirectory);
+    newWindowCmd += ` -c '${expandedPath}'`;
+  }
+
+  await executeTmux(newWindowCmd);
+  const windows = await listWindows(sessionId);
+  const window = windows.find((window) => window.name === name);
+
+  if (!window) return null;
+
+  // Disable automatic window renaming
+  await executeTmux(`set-window-option -t '${window.id}' automatic-rename off`);
+
+  // Get the default pane created with the window
+  const panes = await listPanes(window.id);
+  const defaultPane = panes[0];
+
+  let commandOutput: string | undefined;
+
+  // If command is provided, execute it in the new pane
+  if (options?.command && defaultPane) {
+    await sendText({
+      paneId: defaultPane.id,
+      text: options.command,
+      pressEnter: true,
+    });
+
+    // Sleep for 1 second to allow command to execute
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Capture the pane content
+    commandOutput = await capturePaneContent(defaultPane.id, 200, false);
+  }
+
+  return {
+    ...window,
+    paneId: defaultPane?.id || "",
+    output: commandOutput,
+  };
+}
+
+/**
+ * Kill a tmux session by ID
+ */
+export async function killSession(sessionId: string): Promise<void> {
+  await executeTmux(`kill-session -t '${sessionId}'`);
+}
+
+/**
+ * Kill a tmux window by ID
+ */
+export async function killWindow(windowId: string): Promise<void> {
+  await executeTmux(`kill-window -t '${windowId}'`);
+}
+
+/**
+ * Kill a tmux pane by ID
+ */
+export async function killPane(paneId: string): Promise<void> {
+  await executeTmux(`kill-pane -t '${paneId}'`);
+}
+
+/**
+ * Rename a tmux window by name or ID
+ */
+export async function renameWindow(
+  sessionId: string,
+  windowNameOrId: string,
+  newName: string
+): Promise<void> {
+  // Validate new name is unique
+  const isUnique = await isWindowNameUnique(sessionId, newName);
+  if (!isUnique) {
+    throw new Error(
+      `Terminal with name '${newName}' already exists. Please choose a unique name.`
+    );
+  }
+
+  // Check if windowNameOrId is a window ID (starts with @) or a name
+  let windowId: string;
+  if (windowNameOrId.startsWith("@")) {
+    windowId = windowNameOrId;
+  } else {
+    // Resolve name to ID
+    const window = await findWindowByName(sessionId, windowNameOrId);
+    if (!window) {
+      throw new Error(`Terminal '${windowNameOrId}' not found.`);
+    }
+    windowId = window.id;
+  }
+
+  await executeTmux(`rename-window -t '${windowId}' '${newName}'`);
+  // Disable automatic renaming to preserve the manual name
+  await executeTmux(`set-window-option -t '${windowId}' automatic-rename off`);
+}
+
+/**
+ * Split a tmux pane horizontally or vertically
+ */
+export async function splitPane(
+  targetPaneId: string,
+  direction: "horizontal" | "vertical" = "vertical",
+  size?: number
+): Promise<TmuxPane | null> {
+  // Build the split-window command
+  let splitCommand = "split-window";
+
+  // Add direction flag (-h for horizontal, -v for vertical)
+  if (direction === "horizontal") {
+    splitCommand += " -h";
+  } else {
+    splitCommand += " -v";
+  }
+
+  // Add target pane
+  splitCommand += ` -t '${targetPaneId}'`;
+
+  // Add size if specified (as percentage)
+  if (size !== undefined && size > 0 && size < 100) {
+    splitCommand += ` -p ${size}`;
+  }
+
+  // Execute the split command
+  await executeTmux(splitCommand);
+
+  // Get the window ID from the target pane to list all panes
+  const windowInfo = await executeTmux(
+    `display-message -p -t '${targetPaneId}' '#{window_id}'`
+  );
+
+  // List all panes in the window to find the newly created one
+  const panes = await listPanes(windowInfo);
+
+  // The newest pane is typically the last one in the list
+  return panes.length > 0 ? panes[panes.length - 1] : null;
+}
+
+// Map to track ongoing command executions
+const activeCommands = new Map<string, CommandExecution>();
+
+const startMarkerText = "TMUX_MCP_START";
+const endMarkerPrefix = "TMUX_MCP_DONE_";
+
+// Execute a command in a tmux pane and track its execution
+export async function executeCommand(
+  paneId: string,
+  command: string,
+  rawMode?: boolean,
+  noEnter?: boolean
+): Promise<string> {
+  // Generate unique ID for this command execution
+  const commandId = uuidv4();
+
+  let fullCommand: string;
+  if (rawMode || noEnter) {
+    fullCommand = command;
+  } else {
+    const endMarkerText = getEndMarkerText();
+    fullCommand = `echo "${startMarkerText}"; ${command}; echo "${endMarkerText}"`;
+  }
+
+  // Store command in tracking map
+  activeCommands.set(commandId, {
+    id: commandId,
+    paneId,
+    command,
+    status: "pending",
+    startTime: new Date(),
+    rawMode: rawMode || noEnter,
+  });
+
+  // Send the command to the tmux pane
+  if (noEnter) {
+    // Check if this is a special key or key combination
+    // Special keys in tmux are typically capitalized or have special names
+    const specialKeys = [
+      "Up",
+      "Down",
+      "Left",
+      "Right",
+      "Escape",
+      "Tab",
+      "Enter",
+      "Space",
+      "BSpace",
+      "Delete",
+      "Home",
+      "End",
+      "PageUp",
+      "PageDown",
+      "F1",
+      "F2",
+      "F3",
+      "F4",
+      "F5",
+      "F6",
+      "F7",
+      "F8",
+      "F9",
+      "F10",
+      "F11",
+      "F12",
+      "BTab",
+    ];
+
+    // Split the command into parts to handle combinations like "C-b" or "M-x"
+    const parts = fullCommand.split("-");
+    const isSpecialKey = parts.length === 1 && specialKeys.includes(parts[0]);
+    const isKeyCombo =
+      parts.length > 1 &&
+      (parts[0] === "C" || // Control
+        parts[0] === "M" || // Meta/Alt
+        parts[0] === "S"); // Shift
+
+    if (isSpecialKey || isKeyCombo) {
+      // Send special key or key combination as-is
+      await executeTmux(`send-keys -t '${paneId}' ${fullCommand}`);
+    } else {
+      // For regular text, send each character individually to ensure proper processing
+      // This handles both single characters (like 'q', 'f') and strings (like 'beam')
+      for (const char of fullCommand) {
+        await executeTmux(
+          `send-keys -t '${paneId}' '${char.replace(/'/g, "'\\''")}'`
+        );
+      }
+    }
+  } else {
+    await executeTmux(
+      `send-keys -t '${paneId}' '${fullCommand.replace(/'/g, "'\\''")}' Enter`
+    );
+  }
+
+  return commandId;
+}
+
+export async function checkCommandStatus(
+  commandId: string
+): Promise<CommandExecution | null> {
+  const command = activeCommands.get(commandId);
+  if (!command) return null;
+
+  if (command.status !== "pending") return command;
+
+  const content = await capturePaneContent(command.paneId, 1000);
+
+  if (command.rawMode) {
+    command.result =
+      "Status tracking unavailable for rawMode commands. Use capture-pane to monitor interactive apps instead.";
+    return command;
+  }
+
+  // Find the last occurrence of the markers
+  const startIndex = content.lastIndexOf(startMarkerText);
+  const endIndex = content.lastIndexOf(endMarkerPrefix);
+
+  if (startIndex === -1 || endIndex === -1 || endIndex <= startIndex) {
+    command.result = "Command output could not be captured properly";
+    return command;
+  }
+
+  // Extract exit code from the end marker line
+  const endLine = content.substring(endIndex).split("\n")[0];
+  const endMarkerRegex = new RegExp(`${endMarkerPrefix}(\\d+)`);
+  const exitCodeMatch = endLine.match(endMarkerRegex);
+
+  if (exitCodeMatch) {
+    const exitCode = parseInt(exitCodeMatch[1], 10);
+
+    command.status = exitCode === 0 ? "completed" : "error";
+    command.exitCode = exitCode;
+
+    // Extract output between the start and end markers
+    const outputStart = startIndex + startMarkerText.length;
+    const outputContent = content.substring(outputStart, endIndex).trim();
+
+    command.result = outputContent
+      .substring(outputContent.indexOf("\n") + 1)
+      .trim();
+
+    // Update in map
+    activeCommands.set(commandId, command);
+  }
+
+  return command;
+}
+
+// Get command by ID
+export function getCommand(commandId: string): CommandExecution | null {
+  return activeCommands.get(commandId) || null;
+}
+
+// Get all active command IDs
+export function getActiveCommandIds(): string[] {
+  return Array.from(activeCommands.keys());
+}
+
+// Clean up completed commands older than a certain time
+export function cleanupOldCommands(maxAgeMinutes: number = 60): void {
+  const now = new Date();
+
+  for (const [id, command] of activeCommands.entries()) {
+    const ageMinutes =
+      (now.getTime() - command.startTime.getTime()) / (1000 * 60);
+
+    if (command.status !== "pending" && ageMinutes > maxAgeMinutes) {
+      activeCommands.delete(id);
+    }
+  }
+}
+
+function getEndMarkerText(): string {
+  return shellConfig.type === "fish"
+    ? `${endMarkerPrefix}$status`
+    : `${endMarkerPrefix}$?`;
+}
+
+// New consolidated API functions
+
+export type ListScope = "all" | "sessions" | "session" | "window" | "pane";
+
+interface SessionWithWindows extends TmuxSession {
+  windowDetails?: WindowWithPanes[];
+}
+
+interface WindowWithPanes extends TmuxWindow {
+  paneDetails?: TmuxPane[];
+}
+
+export async function list({
+  scope,
+  target,
+}: {
+  scope: ListScope;
+  target?: string;
+}): Promise<
+  | SessionWithWindows[]
+  | TmuxSession[]
+  | TmuxWindow[]
+  | TmuxPane[]
+  | TmuxSession
+  | TmuxWindow
+  | TmuxPane
+> {
+  if (scope === "all") {
+    const sessions = await listSessions();
+    const sessionsWithDetails: SessionWithWindows[] = [];
+
+    for (const session of sessions) {
+      const windows = await listWindows(session.id);
+      const windowsWithPanes: WindowWithPanes[] = [];
+
+      for (const window of windows) {
+        const panes = await listPanes(window.id);
+        windowsWithPanes.push({
+          ...window,
+          paneDetails: panes,
+        });
+      }
+
+      sessionsWithDetails.push({
+        ...session,
+        windowDetails: windowsWithPanes,
+      });
+    }
+
+    return sessionsWithDetails;
+  }
+
+  if (scope === "sessions") {
+    return listSessions();
+  }
+
+  if (scope === "session") {
+    if (!target) {
+      throw new Error("target is required for scope 'session'");
+    }
+    return listWindows(target);
+  }
+
+  if (scope === "window") {
+    if (!target) {
+      throw new Error("target is required for scope 'window'");
+    }
+    return listPanes(target);
+  }
+
+  if (scope === "pane") {
+    if (!target) {
+      throw new Error("target is required for scope 'pane'");
+    }
+    const windowId = await executeTmux(
+      `display-message -p -t '${target}' '#{window_id}'`
+    );
+    const panes = await listPanes(windowId);
+    const pane = panes.find((p) => p.id === target);
+    if (!pane) {
+      throw new Error(`Pane not found: ${target}`);
+    }
+    return pane;
+  }
+
+  throw new Error(`Invalid scope: ${scope}`);
+}
+
+export type KillScope = "session" | "window" | "pane";
+
+export async function kill({
+  scope,
+  target,
+}: {
+  scope: KillScope;
+  target: string;
+}): Promise<void> {
+  if (scope === "session") {
+    return killSession(target);
+  }
+
+  if (scope === "window") {
+    return killWindow(target);
+  }
+
+  if (scope === "pane") {
+    return killPane(target);
+  }
+
+  throw new Error(`Invalid scope: ${scope}`);
+}
+
+export interface ShellCommandResult {
+  command: string;
+  status: "completed" | "error";
+  exitCode: number;
+  output: string;
+}
+
+export async function executeShellCommand({
+  paneId,
+  command,
+  timeout = 30000,
+}: {
+  paneId: string;
+  command: string;
+  timeout?: number;
+}): Promise<ShellCommandResult> {
+  const commandId = uuidv4();
+  const endMarkerText = getEndMarkerText();
+  const fullCommand = `echo "${startMarkerText}"; ${command}; echo "${endMarkerText}"`;
+
+  activeCommands.set(commandId, {
+    id: commandId,
+    paneId,
+    command,
+    status: "pending",
+    startTime: new Date(),
+    rawMode: false,
+  });
+
+  await executeTmux(
+    `send-keys -t '${paneId}' '${fullCommand.replace(/'/g, "'\\''")}' Enter`
+  );
+
+  // Poll for completion
+  const startTime = Date.now();
+  const pollInterval = 100;
+
+  while (Date.now() - startTime < timeout) {
+    await new Promise((resolve) => setTimeout(resolve, pollInterval));
+
+    const result = await checkCommandStatus(commandId);
+
+    if (result && result.status !== "pending") {
+      // Cleanup
+      activeCommands.delete(commandId);
+
+      return {
+        command: result.command,
+        status: result.status,
+        exitCode: result.exitCode!,
+        output: result.result || "",
+      };
+    }
+  }
+
+  // Timeout
+  activeCommands.delete(commandId);
+  throw new Error(
+    `Command timed out after ${timeout}ms. Use capture-pane to check pane state.`
+  );
+}
+
+export async function sendKeys({
+  paneId,
+  keys,
+  repeat = 1,
+  return_output,
+}: {
+  paneId: string;
+  keys: string;
+  repeat?: number;
+  return_output?: { lines?: number; wait?: number };
+}): Promise<string | void> {
+  // Repeat the key press the specified number of times
+  for (let i = 0; i < repeat; i++) {
+    // Raw pass-through, no validation or processing
+    await executeTmux(`send-keys -t '${paneId}' ${keys}`);
+  }
+
+  // If return_output is requested, wait and capture pane content
+  if (return_output) {
+    await new Promise((resolve) =>
+      setTimeout(resolve, return_output.wait ?? 500)
+    );
+    const lines = return_output.lines || 200;
+    return capturePaneContent(paneId, lines, false);
+  }
+}
+
+export async function sendText({
+  paneId,
+  text,
+  pressEnter = false,
+  return_output,
+}: {
+  paneId: string;
+  text: string;
+  pressEnter?: boolean;
+  return_output?: { lines?: number; wait?: number };
+}): Promise<string | void> {
+  // Send each character with -l flag for literal interpretation
+  for (const char of text) {
+    await executeTmux(
+      `send-keys -l -t '${paneId}' '${char.replace(/'/g, "'\\''")}'`
+    );
+  }
+
+  if (pressEnter) {
+    await new Promise((resolve) => setTimeout(resolve, 300));
+    await executeTmux(`send-keys -t '${paneId}' Enter`);
+  }
+
+  // If return_output is requested, wait and capture pane content
+  if (return_output) {
+    if (return_output.wait) {
+      await new Promise((resolve) => setTimeout(resolve, return_output.wait));
+    }
+    const lines = return_output.lines || 200;
+    return capturePaneContent(paneId, lines, false);
+  }
+}

packages/voice-assistant/src/server/daemon/tool-definitions.ts

@@ -0,0 +1,166 @@
+/**
+ * OpenAI function calling schemas for terminal operations
+ * These tools are exposed to the LLM for terminal control
+ */
+
+export const terminalTools = [
+  {
+    type: "function",
+    function: {
+      name: "list_terminals",
+      description:
+        "List all available terminals in the voice-dev session. Returns terminal ID, name, working directory, and current command for each terminal.",
+      parameters: {
+        type: "object",
+        properties: {},
+        required: [],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "create_terminal",
+      description:
+        "Create a new terminal (tmux window) with a specific name and working directory. Optionally execute an initial command after creation. Terminal names must be unique.",
+      parameters: {
+        type: "object",
+        properties: {
+          name: {
+            type: "string",
+            description:
+              "Unique name for the terminal. Should be descriptive of what the terminal is used for (e.g., 'web-dev', 'api-server', 'tests').",
+          },
+          workingDirectory: {
+            type: "string",
+            description:
+              "Absolute path to the working directory for this terminal. Can use ~ for home directory.",
+          },
+          initialCommand: {
+            type: "string",
+            description:
+              "Optional command to execute after creating the terminal (e.g., 'npm run dev', 'python -m venv venv').",
+          },
+        },
+        required: ["name", "workingDirectory"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "capture_terminal",
+      description:
+        "Capture and return the output from a terminal. Returns the last N lines of terminal content. Useful for checking command results or monitoring running processes.",
+      parameters: {
+        type: "object",
+        properties: {
+          terminalId: {
+            type: "string",
+            description:
+              "The terminal ID (window ID format: @123) to capture output from. Get this from list_terminals.",
+          },
+          lines: {
+            type: "number",
+            description:
+              "Number of lines to capture from the terminal history. Defaults to 200.",
+          },
+          wait: {
+            type: "number",
+            description:
+              "Optional milliseconds to wait before capturing output. Useful when you just sent a command and want to wait for it to complete.",
+          },
+        },
+        required: ["terminalId"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "send_text",
+      description:
+        "Type text into a terminal. Can optionally press Enter to execute the text as a command. Can optionally return the output after execution. This is the primary way to run commands in terminals.",
+      parameters: {
+        type: "object",
+        properties: {
+          terminalId: {
+            type: "string",
+            description:
+              "The terminal ID (window ID format: @123) to send text to. Get this from list_terminals.",
+          },
+          text: {
+            type: "string",
+            description:
+              "The text to type into the terminal. Can be a command, input for a running program, or any text.",
+          },
+          pressEnter: {
+            type: "boolean",
+            description:
+              "Whether to press Enter after typing the text. Set to true to execute commands. Defaults to false.",
+          },
+          return_output: {
+            type: "object",
+            description:
+              "Optional. If provided, will wait and return the terminal output after sending the text.",
+            properties: {
+              lines: {
+                type: "number",
+                description: "Number of lines to capture. Defaults to 200.",
+              },
+              wait: {
+                type: "number",
+                description:
+                  "Milliseconds to wait before capturing output. Defaults to 500ms.",
+              },
+            },
+          },
+        },
+        required: ["terminalId", "text"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "rename_terminal",
+      description:
+        "Rename an existing terminal. The new name must be unique among all terminals.",
+      parameters: {
+        type: "object",
+        properties: {
+          terminalId: {
+            type: "string",
+            description:
+              "The terminal ID (window ID format: @123) to rename. Get this from list_terminals.",
+          },
+          newName: {
+            type: "string",
+            description:
+              "The new unique name for the terminal. Should be descriptive of the terminal's purpose.",
+          },
+        },
+        required: ["terminalId", "newName"],
+      },
+    },
+  },
+  {
+    type: "function",
+    function: {
+      name: "kill_terminal",
+      description:
+        "Close and destroy a terminal. This will terminate any running processes in the terminal. Use with caution.",
+      parameters: {
+        type: "object",
+        properties: {
+          terminalId: {
+            type: "string",
+            description:
+              "The terminal ID (window ID format: @123) to kill. Get this from list_terminals.",
+          },
+        },
+        required: ["terminalId"],
+      },
+    },
+  },
+] as const;

packages/voice-assistant/src/server/index.ts

@@ -0,0 +1,336 @@
+import express from 'express';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { createServer as createHTTPServer } from 'http';
+import { readFile } from 'fs/promises';
+import { v4 as uuidv4 } from 'uuid';
+import { createServer as createViteServer } from 'vite';
+import type { ViteDevServer } from 'vite';
+import type { ServerConfig } from './types.js';
+import { VoiceAssistantWebSocketServer } from './websocket-server.js';
+import {
+  initializeDefaultSession,
+  listTerminals,
+  createTerminal,
+  sendText,
+  captureTerminal,
+  killTerminal,
+} from './daemon/terminal-manager.js';
+import { initializeOpenAI } from './agent/llm-openai.js';
+import { initializeSTT, transcribeAudio } from './agent/stt-openai.js';
+import { initializeTTS, synthesizeSpeech } from './agent/tts-openai.js';
+import {
+  createConversation,
+  processUserMessage,
+  cleanupConversations,
+} from './agent/orchestrator.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+async function createServer(config: ServerConfig) {
+  const app = express();
+
+  // Middleware
+  app.use(express.json());
+
+  // Health check endpoint
+  app.get('/api/health', (_req, res) => {
+    res.json({ status: 'ok', timestamp: new Date().toISOString() });
+  });
+
+  // Test endpoint for LLM integration (removed - using orchestrator now)
+
+  // Test endpoint for terminal operations
+  app.get('/api/test-terminal', async (_req, res) => {
+    try {
+      const testResults: string[] = [];
+
+      // 1. Initialize default session
+      testResults.push('1. Initializing default session...');
+      await initializeDefaultSession();
+      testResults.push('   ✓ Session initialized');
+
+      // 2. List existing terminals
+      testResults.push('2. Listing existing terminals...');
+      const beforeTerminals = await listTerminals();
+      testResults.push(`   ✓ Found ${beforeTerminals.length} existing terminals`);
+
+      // 3. Create a test terminal
+      testResults.push('3. Creating test terminal...');
+      const terminal = await createTerminal({
+        name: 'test-terminal',
+        workingDirectory: '~',
+        initialCommand: undefined,
+      });
+      testResults.push(`   ✓ Created terminal: ${terminal.id} (${terminal.name})`);
+
+      // 4. Send a command to the terminal
+      testResults.push('4. Sending command "echo hello world"...');
+      await sendText(terminal.id, 'echo "hello world"', true, { lines: 20, wait: 500 });
+      testResults.push('   ✓ Command sent');
+
+      // 5. Capture output
+      testResults.push('5. Capturing terminal output...');
+      const output = await captureTerminal(terminal.id, 20);
+      testResults.push(`   ✓ Captured ${output.split('\n').length} lines`);
+      testResults.push(`   Output preview: ${output.substring(0, 100)}...`);
+
+      // 6. List terminals again
+      testResults.push('6. Listing terminals again...');
+      const afterTerminals = await listTerminals();
+      testResults.push(`   ✓ Found ${afterTerminals.length} terminals`);
+      const testTerminal = afterTerminals.find((t) => t.name === 'test-terminal');
+      if (testTerminal) {
+        testResults.push(`   ✓ Found test-terminal: ${testTerminal.id}`);
+      }
+
+      // 7. Kill the test terminal
+      testResults.push('7. Killing test terminal...');
+      await killTerminal(terminal.id);
+      testResults.push('   ✓ Terminal killed');
+
+      // 8. Verify it's gone
+      testResults.push('8. Verifying terminal was removed...');
+      const finalTerminals = await listTerminals();
+      const stillExists = finalTerminals.find((t) => t.name === 'test-terminal');
+      if (!stillExists) {
+        testResults.push('   ✓ Terminal successfully removed');
+      } else {
+        testResults.push('   ✗ Terminal still exists!');
+      }
+
+      res.json({
+        success: true,
+        message: 'All terminal operations completed successfully',
+        results: testResults,
+      });
+    } catch (error: any) {
+      res.status(500).json({
+        success: false,
+        error: error.message,
+        stack: error.stack,
+      });
+    }
+  });
+
+  let vite: ViteDevServer | undefined;
+
+  if (config.isDev) {
+    // Development: Create Vite server in middleware mode
+    vite = await createViteServer({
+      server: { middlewareMode: true },
+      appType: 'custom',
+    });
+
+    // Use Vite's middleware to handle HMR, module transformation, etc.
+    app.use(vite.middlewares);
+  } else {
+    // Production: Serve static files from dist/ui
+    const uiPath = path.join(__dirname, '../ui');
+    app.use(express.static(uiPath));
+  }
+
+  // SPA fallback - serve index.html for all non-API routes
+  app.use('*', async (req, res, next) => {
+    if (req.originalUrl.startsWith('/api')) {
+      return res.status(404).json({ error: 'Not found' });
+    }
+
+    try {
+      const indexHtmlPath = config.isDev
+        ? path.resolve(__dirname, '../../src/ui/index.html')
+        : path.join(__dirname, '../ui/index.html');
+
+      let html = await readFile(indexHtmlPath, 'utf-8');
+
+      // In development, transform the HTML with Vite
+      if (config.isDev && vite) {
+        html = await vite.transformIndexHtml(req.originalUrl, html);
+      }
+
+      res.status(200).set({ 'Content-Type': 'text/html' }).end(html);
+    } catch (e) {
+      if (config.isDev && vite) {
+        vite.ssrFixStacktrace(e as Error);
+      }
+      next(e);
+    }
+  });
+
+  return { app, vite };
+}
+
+async function main() {
+  const port = parseInt(process.env.PORT || '3000', 10);
+  const isDev = process.env.NODE_ENV !== 'production';
+
+  const config: ServerConfig = { port, isDev };
+  const { app, vite } = await createServer(config);
+
+  // Create HTTP server
+  const httpServer = createHTTPServer(app);
+
+  // Initialize WebSocket server
+  const wsServer = new VoiceAssistantWebSocketServer(httpServer);
+
+  // Initialize OpenAI client
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (apiKey) {
+    initializeOpenAI(apiKey);
+    console.log('✓ OpenAI client initialized');
+
+    // Initialize STT (Whisper)
+    initializeSTT({ apiKey });
+
+    // Initialize TTS
+    const ttsVoice = (process.env.TTS_VOICE || 'alloy') as 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
+    const ttsModel = (process.env.TTS_MODEL || 'tts-1') as 'tts-1' | 'tts-1-hd';
+    initializeTTS({
+      apiKey,
+      voice: ttsVoice,
+      model: ttsModel,
+      responseFormat: 'mp3',
+    });
+  } else {
+    console.warn('⚠ OPENAI_API_KEY not set - LLM, STT, and TTS features will not work');
+  }
+
+  // Create default conversation
+  const defaultConversationId = createConversation();
+  console.log(`✓ Default conversation created: ${defaultConversationId}`);
+
+  // Helper function to process message and optionally generate TTS
+  async function processMessageWithOptionalTTS(message: string, enableTTS: boolean = false): Promise<string> {
+    const response = await processUserMessage({
+      conversationId: defaultConversationId,
+      message,
+      wsServer,
+    });
+
+    // Generate TTS for the response if enabled
+    if (enableTTS && apiKey) {
+      try {
+        console.log('[TTS] Generating speech for assistant response...');
+        const speechResult = await synthesizeSpeech(response);
+
+        // Convert audio buffer to base64
+        const base64Audio = speechResult.audio.toString('base64');
+
+        // Send audio to client via WebSocket
+        wsServer.broadcast({
+          type: 'audio_output',
+          payload: {
+            id: uuidv4(),
+            audio: base64Audio,
+            format: speechResult.format,
+          },
+        });
+
+        console.log(`[TTS] Sent audio to client via WebSocket (${speechResult.audio.length} bytes)`);
+      } catch (error: any) {
+        console.error('[TTS] Error generating speech:', error);
+        wsServer.broadcastActivityLog({
+          id: uuidv4(),
+          timestamp: new Date(),
+          type: 'error',
+          content: `TTS error: ${error.message}`,
+        });
+      }
+    }
+
+    return response;
+  }
+
+  // Wire orchestrator to WebSocket for text messages
+  wsServer.setMessageHandler(async (message: string) => {
+    try {
+      await processUserMessage({
+        conversationId: defaultConversationId,
+        message,
+        wsServer,
+      });
+    } catch (error: any) {
+      console.error('[Orchestrator] Error processing message:', error);
+      wsServer.broadcastActivityLog({
+        id: uuidv4(),
+        timestamp: new Date(),
+        type: 'error',
+        content: `Error: ${error.message}`,
+      });
+    }
+  });
+
+  // Wire audio handler to WebSocket for voice input (STT)
+  wsServer.setAudioHandler(async (audio: Buffer, format: string): Promise<string> => {
+    try {
+      // Transcribe audio using OpenAI Whisper
+      const result = await transcribeAudio(audio, format);
+
+      // Broadcast transcription result as activity log
+      wsServer.broadcastActivityLog({
+        id: uuidv4(),
+        timestamp: new Date(),
+        type: 'transcript',
+        content: result.text,
+        metadata: {
+          language: result.language,
+          duration: result.duration,
+        },
+      });
+
+      // Process the transcribed text through the orchestrator WITH TTS enabled
+      // Since this came from voice input, respond with voice output
+      await processMessageWithOptionalTTS(result.text, true);
+
+      return result.text;
+    } catch (error: any) {
+      console.error('[STT] Error transcribing audio:', error);
+      wsServer.broadcastActivityLog({
+        id: uuidv4(),
+        timestamp: new Date(),
+        type: 'error',
+        content: `Transcription error: ${error.message}`,
+      });
+      throw error;
+    }
+  });
+
+  // Start conversation cleanup interval (every 10 minutes)
+  setInterval(() => {
+    cleanupConversations(60); // Clean up conversations older than 60 minutes
+  }, 10 * 60 * 1000);
+
+  // Initialize default tmux session for terminal control
+  try {
+    await initializeDefaultSession();
+    console.log('✓ Default tmux session initialized');
+  } catch (error) {
+    console.error('Failed to initialize tmux session:', error);
+  }
+
+  httpServer.listen(port, () => {
+    console.log(`\n✓ Voice Assistant server running on http://localhost:${port}`);
+    if (isDev) {
+      console.log(`✓ Vite dev server integrated (HMR enabled)\n`);
+    }
+  });
+
+  // Graceful shutdown
+  process.on('SIGTERM', async () => {
+    console.log('\nSIGTERM received, shutting down gracefully...');
+    wsServer.close();
+    if (vite) {
+      await vite.close();
+    }
+    httpServer.close(() => {
+      console.log('Server closed');
+      process.exit(0);
+    });
+  });
+}
+
+main().catch((error) => {
+  console.error('Failed to start server:', error);
+  process.exit(1);
+});

packages/voice-assistant/src/server/types.ts

@@ -0,0 +1,42 @@
+// Shared server types
+
+export interface ServerConfig {
+  port: number;
+  isDev: boolean;
+}
+
+export interface ActivityLogEntry {
+  id: string;
+  timestamp: Date;
+  type: 'transcript' | 'assistant' | 'tool_call' | 'tool_result' | 'error' | 'system';
+  content: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface WebSocketMessage {
+  type: 'activity_log' | 'status' | 'ping' | 'pong' | 'user_message' | 'assistant_chunk'
+       | 'audio_chunk' | 'audio_output' | 'recording_state' | 'transcription_result';
+  payload: unknown;
+}
+
+export interface AudioChunkPayload {
+  audio: string; // base64 encoded audio data
+  format: string; // 'webm', 'ogg', etc.
+  isLast: boolean; // true when recording stopped
+}
+
+export interface RecordingStatePayload {
+  isRecording: boolean;
+}
+
+export interface TranscriptionResultPayload {
+  text: string;
+  language?: string;
+  duration?: number;
+}
+
+export interface AudioOutputPayload {
+  audio: string; // base64 encoded audio data (complete)
+  format: string; // 'mp3'
+  id: string; // unique ID for queue management
+}

packages/voice-assistant/src/server/websocket-server.ts

@@ -0,0 +1,197 @@
+import { WebSocketServer, WebSocket } from 'ws';
+import { Server as HTTPServer } from 'http';
+import type {
+  WebSocketMessage,
+  ActivityLogEntry,
+  AudioChunkPayload,
+} from './types.js';
+
+export class VoiceAssistantWebSocketServer {
+  private wss: WebSocketServer;
+  private clients: Map<WebSocket, string> = new Map(); // Map ws to client ID
+  private messageHandler?: (message: string) => Promise<void>;
+  private audioHandler?: (audio: Buffer, format: string) => Promise<string>;
+  private audioBuffers: Map<string, { chunks: Buffer[]; format: string }> = new Map();
+  private clientIdCounter: number = 0;
+
+  constructor(server: HTTPServer) {
+    this.wss = new WebSocketServer({ server, path: '/ws' });
+
+    this.wss.on('connection', (ws) => {
+      this.handleConnection(ws);
+    });
+
+    console.log('✓ WebSocket server initialized on /ws');
+  }
+
+  private handleConnection(ws: WebSocket): void {
+    // Generate unique client ID
+    const clientId = `client-${++this.clientIdCounter}`;
+    this.clients.set(ws, clientId);
+    console.log(`[WS] Client connected: ${clientId} (total: ${this.clients.size})`);
+
+    // Send welcome message
+    this.sendToClient(ws, {
+      type: 'status',
+      payload: { status: 'connected', message: 'WebSocket connection established' },
+    });
+
+    ws.on('message', (data) => {
+      this.handleMessage(ws, data);
+    });
+
+    ws.on('close', () => {
+      const clientId = this.clients.get(ws);
+      if (clientId) {
+        this.clients.delete(ws);
+        console.log(`[WS] Client disconnected: ${clientId} (total: ${this.clients.size})`);
+      }
+    });
+
+    ws.on('error', (error) => {
+      console.error('[WS] Client error:', error);
+      const clientId = this.clients.get(ws);
+      if (clientId) {
+        this.clients.delete(ws);
+      }
+    });
+  }
+
+  private async handleMessage(ws: WebSocket, data: Buffer | ArrayBuffer | Buffer[]): Promise<void> {
+    try {
+      const message = JSON.parse(data.toString()) as WebSocketMessage;
+
+      console.log(`[WS] Received message type: ${message.type}`);
+
+      switch (message.type) {
+        case 'ping':
+          this.sendToClient(ws, { type: 'pong', payload: {} });
+          break;
+
+        case 'user_message':
+          // Handle user message through orchestrator
+          const payload = message.payload as { message: string };
+          if (this.messageHandler) {
+            await this.messageHandler(payload.message);
+          } else {
+            console.warn('[WS] No message handler registered');
+          }
+          break;
+
+        case 'audio_chunk':
+          // Handle audio chunk for STT
+          await this.handleAudioChunk(ws, message.payload as AudioChunkPayload);
+          break;
+
+        default:
+          console.warn(`[WS] Unknown message type: ${message.type}`);
+      }
+    } catch (error) {
+      console.error('[WS] Failed to parse message:', error);
+    }
+  }
+
+  private sendToClient(ws: WebSocket, message: WebSocketMessage): void {
+    if (ws.readyState === 1) { // WebSocket.OPEN = 1
+      ws.send(JSON.stringify(message));
+    }
+  }
+
+  public broadcast(message: WebSocketMessage): void {
+    const payload = JSON.stringify(message);
+    this.clients.forEach((_clientId, client) => {
+      if (client.readyState === 1) { // WebSocket.OPEN = 1
+        client.send(payload);
+      }
+    });
+  }
+
+  public broadcastActivityLog(entry: ActivityLogEntry): void {
+    this.broadcast({
+      type: 'activity_log',
+      payload: entry,
+    });
+  }
+
+  public broadcastStatus(status: string, metadata?: Record<string, unknown>): void {
+    this.broadcast({
+      type: 'status',
+      payload: { status, ...metadata },
+    });
+  }
+
+  public setMessageHandler(handler: (message: string) => Promise<void>): void {
+    this.messageHandler = handler;
+  }
+
+  public setAudioHandler(handler: (audio: Buffer, format: string) => Promise<string>): void {
+    this.audioHandler = handler;
+  }
+
+  private async handleAudioChunk(ws: WebSocket, payload: AudioChunkPayload): Promise<void> {
+    try {
+      // Use a client-specific key for buffering (in case multiple clients)
+      const clientId = 'default'; // Could be enhanced with per-client tracking
+
+      // Decode base64 audio data
+      const audioBuffer = Buffer.from(payload.audio, 'base64');
+
+      if (!payload.isLast) {
+        // Buffer the chunk
+        if (!this.audioBuffers.has(clientId)) {
+          this.audioBuffers.set(clientId, { chunks: [], format: payload.format });
+        }
+        const buffer = this.audioBuffers.get(clientId)!;
+        buffer.chunks.push(audioBuffer);
+        console.log(`[WS] Buffered audio chunk (${audioBuffer.length} bytes, total chunks: ${buffer.chunks.length})`);
+      } else {
+        // Last chunk - process complete audio
+        const buffer = this.audioBuffers.get(clientId);
+        const allChunks = buffer ? [...buffer.chunks, audioBuffer] : [audioBuffer];
+        const format = buffer?.format || payload.format;
+
+        // Concatenate all chunks
+        const completeAudio = Buffer.concat(allChunks);
+        console.log(`[WS] Complete audio received (${completeAudio.length} bytes, ${allChunks.length} chunks)`);
+
+        // Clear buffer
+        this.audioBuffers.delete(clientId);
+
+        // Process audio through handler (STT)
+        if (this.audioHandler) {
+          this.broadcastActivityLog({
+            id: Date.now().toString(),
+            timestamp: new Date(),
+            type: 'system',
+            content: 'Transcribing audio...',
+          });
+
+          const transcript = await this.audioHandler(completeAudio, format);
+
+          // Send transcription result back to client
+          this.sendToClient(ws, {
+            type: 'transcription_result',
+            payload: { text: transcript },
+          });
+        } else {
+          console.warn('[WS] No audio handler registered');
+        }
+      }
+    } catch (error: any) {
+      console.error('[WS] Audio chunk handling error:', error);
+      this.broadcastActivityLog({
+        id: Date.now().toString(),
+        timestamp: new Date(),
+        type: 'error',
+        content: `Audio processing error: ${error.message}`,
+      });
+    }
+  }
+
+  public close(): void {
+    this.clients.forEach((_clientId, ws) => {
+      ws.close();
+    });
+    this.wss.close();
+  }
+}

packages/voice-assistant/src/ui/App.css

@@ -0,0 +1,391 @@
+.app {
+  display: flex;
+  flex-direction: column;
+  min-height: 100vh;
+  width: 100%;
+}
+
+.header {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  padding: 1rem 2rem;
+  background-color: #1a1a1a;
+  border-bottom: 1px solid #333;
+}
+
+.header h1 {
+  font-size: 1.5rem;
+  font-weight: 600;
+}
+
+.status-indicator {
+  padding: 0.5rem 1rem;
+  border-radius: 4px;
+  font-size: 0.75rem;
+  font-weight: 500;
+  text-transform: uppercase;
+}
+
+.status-indicator.disconnected {
+  background-color: #444;
+  color: #999;
+}
+
+.status-indicator.connecting {
+  background-color: #ffa500;
+  color: #000;
+  animation: pulse 1.5s ease-in-out infinite;
+}
+
+.status-indicator.connected {
+  background-color: #22c55e;
+  color: #000;
+}
+
+@keyframes pulse {
+  0%, 100% {
+    opacity: 1;
+  }
+  50% {
+    opacity: 0.6;
+  }
+}
+
+.main {
+  display: flex;
+  flex: 1;
+  padding: 2rem;
+  gap: 2rem;
+}
+
+.chat-interface {
+  flex: 1;
+  background-color: #1a1a1a;
+  border-radius: 8px;
+  padding: 2rem;
+  display: flex;
+  flex-direction: column;
+  gap: 1rem;
+  max-height: calc(100vh - 120px);
+}
+
+.chat-interface h2 {
+  font-size: 1.25rem;
+  font-weight: 600;
+  margin: 0;
+  padding-bottom: 0.75rem;
+  border-bottom: 1px solid #333;
+}
+
+.ping-button {
+  padding: 0.75rem 2rem;
+  font-size: 1rem;
+  font-weight: 500;
+  background-color: #22c55e;
+  color: #000;
+  border: none;
+  border-radius: 6px;
+  cursor: pointer;
+  transition: background-color 0.2s;
+}
+
+.ping-button:hover:not(:disabled) {
+  background-color: #16a34a;
+}
+
+.ping-button:disabled {
+  background-color: #444;
+  color: #999;
+  cursor: not-allowed;
+}
+
+.connection-info {
+  color: #999;
+  font-size: 0.875rem;
+  text-align: center;
+  max-width: 400px;
+}
+
+.activity-log {
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+}
+
+.log-entries {
+  flex: 1;
+  overflow-y: auto;
+  display: flex;
+  flex-direction: column;
+  gap: 0.5rem;
+}
+
+.log-entry {
+  padding: 0.75rem;
+  border-radius: 4px;
+  font-size: 0.875rem;
+  line-height: 1.4;
+  display: flex;
+  gap: 0.75rem;
+}
+
+.log-time {
+  font-family: monospace;
+  color: #666;
+  flex-shrink: 0;
+}
+
+.log-message {
+  flex: 1;
+}
+
+.log-entry.system {
+  background-color: #2a2a2a;
+  color: #999;
+  font-style: italic;
+}
+
+.log-entry.info {
+  background-color: #1e3a8a;
+  color: #93c5fd;
+}
+
+.log-entry.success {
+  background-color: #14532d;
+  color: #86efac;
+}
+
+.log-entry.error {
+  background-color: #7f1d1d;
+  color: #fca5a5;
+}
+
+.log-entry.user {
+  background-color: #1e293b;
+  color: #e2e8f0;
+}
+
+.log-entry.assistant {
+  background-color: #0f766e;
+  color: #ccfbf1;
+}
+
+.log-entry.tool {
+  background-color: #581c87;
+  color: #e9d5ff;
+  font-family: monospace;
+  font-size: 0.75rem;
+}
+
+.log-entry.streaming {
+  opacity: 0.8;
+}
+
+.streaming-indicator {
+  animation: dots 1.5s steps(3, end) infinite;
+  color: #666;
+}
+
+@keyframes dots {
+  0%, 20% { content: '.'; }
+  40% { content: '..'; }
+  60%, 100% { content: '...'; }
+}
+
+.log-metadata {
+  margin-top: 0.5rem;
+}
+
+.log-metadata summary {
+  cursor: pointer;
+  color: #999;
+  font-size: 0.75rem;
+}
+
+.log-metadata pre {
+  margin-top: 0.5rem;
+  padding: 0.5rem;
+  background-color: #000;
+  border-radius: 4px;
+  overflow-x: auto;
+  font-size: 0.75rem;
+}
+
+.message-form {
+  display: flex;
+  gap: 0.5rem;
+  padding-top: 1rem;
+  border-top: 1px solid #333;
+}
+
+.message-input {
+  flex: 1;
+  padding: 0.75rem 1rem;
+  background-color: #2a2a2a;
+  border: 1px solid #444;
+  border-radius: 6px;
+  color: #fff;
+  font-size: 0.875rem;
+  outline: none;
+}
+
+.message-input:focus {
+  border-color: #22c55e;
+}
+
+.message-input:disabled {
+  opacity: 0.5;
+  cursor: not-allowed;
+}
+
+.send-button {
+  padding: 0.75rem 1.5rem;
+  background-color: #22c55e;
+  color: #000;
+  border: none;
+  border-radius: 6px;
+  font-size: 0.875rem;
+  font-weight: 500;
+  cursor: pointer;
+  transition: background-color 0.2s;
+}
+
+.send-button:hover:not(:disabled) {
+  background-color: #16a34a;
+}
+
+.send-button:disabled {
+  background-color: #444;
+  color: #999;
+  cursor: not-allowed;
+}
+
+.test-controls {
+  padding-top: 0.5rem;
+  border-top: 1px solid #333;
+  display: flex;
+  justify-content: center;
+}
+
+/* Voice Controls */
+.voice-controls {
+  display: flex;
+  flex-direction: column;
+  gap: 0.75rem;
+  padding: 1rem;
+  background-color: #2a2a2a;
+  border-radius: 6px;
+  margin-bottom: 1rem;
+}
+
+.voice-button {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 0.5rem;
+  padding: 1rem 2rem;
+  background-color: #3b82f6;
+  color: #fff;
+  border: none;
+  border-radius: 8px;
+  font-size: 1rem;
+  font-weight: 500;
+  cursor: pointer;
+  transition: all 0.2s;
+}
+
+.voice-button:hover:not(:disabled) {
+  background-color: #2563eb;
+  transform: scale(1.02);
+}
+
+.voice-button:disabled {
+  background-color: #444;
+  color: #999;
+  cursor: not-allowed;
+  transform: none;
+}
+
+.voice-button.recording {
+  background-color: #ef4444;
+  animation: pulse-red 1.5s ease-in-out infinite;
+}
+
+.voice-button.recording:hover:not(:disabled) {
+  background-color: #dc2626;
+}
+
+.voice-button.processing {
+  background-color: #f59e0b;
+}
+
+@keyframes pulse-red {
+  0%, 100% {
+    opacity: 1;
+  }
+  50% {
+    opacity: 0.7;
+  }
+}
+
+.recording-indicator {
+  display: inline-block;
+  width: 12px;
+  height: 12px;
+  background-color: #fff;
+  border-radius: 50%;
+  animation: pulse 1s ease-in-out infinite;
+}
+
+.processing-indicator {
+  display: inline-block;
+  width: 12px;
+  height: 12px;
+  border: 2px solid #fff;
+  border-top-color: transparent;
+  border-radius: 50%;
+  animation: spin 0.8s linear infinite;
+}
+
+@keyframes spin {
+  to {
+    transform: rotate(360deg);
+  }
+}
+
+.microphone-icon {
+  font-size: 1.25rem;
+}
+
+.voice-error {
+  padding: 0.75rem;
+  background-color: #7f1d1d;
+  color: #fca5a5;
+  border-radius: 4px;
+  font-size: 0.875rem;
+}
+
+.permission-denied {
+  padding: 0.75rem;
+  background-color: #7c2d12;
+  color: #fed7aa;
+  border-radius: 4px;
+  font-size: 0.875rem;
+}
+
+.playing-indicator {
+  padding: 0.5rem;
+  background-color: #0f766e;
+  color: #ccfbf1;
+  border-radius: 4px;
+  font-size: 0.875rem;
+  text-align: center;
+}
+
+@media (max-width: 1024px) {
+  .main {
+    flex-direction: column;
+  }
+}

packages/voice-assistant/src/ui/App.tsx

@@ -0,0 +1,260 @@
+import { useState, useEffect, useRef } from 'react';
+import { useWebSocket } from './hooks/useWebSocket';
+import { VoiceControls } from './components/VoiceControls';
+import { createAudioPlayer } from './lib/audio-playback';
+import './App.css';
+
+interface LogEntry {
+  id: string;
+  timestamp: number;
+  type: 'system' | 'info' | 'success' | 'error' | 'user' | 'assistant' | 'tool';
+  message: string;
+  metadata?: Record<string, unknown>;
+}
+
+function App() {
+  const [logs, setLogs] = useState<LogEntry[]>([
+    { id: '1', timestamp: Date.now(), type: 'system', message: 'System initialized' },
+  ]);
+  const [userInput, setUserInput] = useState('');
+  const [currentAssistantMessage, setCurrentAssistantMessage] = useState('');
+  const [isProcessingAudio, setIsProcessingAudio] = useState(false);
+  const [isPlayingAudio, setIsPlayingAudio] = useState(false);
+  const logEndRef = useRef<HTMLDivElement>(null);
+  const audioPlayerRef = useRef(createAudioPlayer());
+
+  // WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod
+  const wsUrl = `ws://${window.location.host}/ws`;
+
+  const ws = useWebSocket(wsUrl);
+
+  const addLog = (
+    type: LogEntry['type'],
+    message: string,
+    metadata?: Record<string, unknown>
+  ) => {
+    console.trace("addLog", type, message, metadata);
+    setLogs((prev) => [
+      ...prev,
+      { id: Date.now().toString(), timestamp: Date.now(), type, message, metadata },
+    ]);
+  };
+
+  useEffect(() => {
+    // Listen for status messages
+    const unsubStatus = ws.on('status', (payload: unknown) => {
+      const data = payload as { status: string; message?: string };
+      addLog('info', data.message || `Status: ${data.status}`);
+    });
+
+    // Listen for pong messages
+    const unsubPong = ws.on('pong', () => {
+      addLog('success', 'Received pong from server');
+    });
+
+    // Listen for activity log messages
+    const unsubActivity = ws.on('activity_log', (payload: unknown) => {
+      const data = payload as {
+        type: string;
+        content: string;
+        metadata?: Record<string, unknown>;
+      };
+
+      // Map activity log types to UI log types
+      let logType: LogEntry['type'] = 'info';
+      if (data.type === 'transcript') logType = 'user';
+      else if (data.type === 'assistant') logType = 'assistant';
+      else if (data.type === 'tool_call' || data.type === 'tool_result') logType = 'tool';
+      else if (data.type === 'error') logType = 'error';
+
+      addLog(logType, data.content, data.metadata);
+    });
+
+    // Listen for streaming assistant chunks
+    const unsubChunk = ws.on('assistant_chunk', (payload: unknown) => {
+      const data = payload as { chunk: string };
+      setCurrentAssistantMessage((prev) => prev + data.chunk);
+    });
+
+    // Listen for transcription results
+    const unsubTranscription = ws.on('transcription_result', (payload: unknown) => {
+      const data = payload as { text: string };
+      addLog('info', `Transcription: ${data.text}`);
+      setIsProcessingAudio(false);
+    });
+
+    // Listen for audio output (TTS)
+    const unsubAudioOutput = ws.on('audio_output', async (payload: unknown) => {
+      const data = payload as { audio: string; format: string; id: string };
+
+      try {
+        addLog('info', 'Playing assistant audio response...');
+        setIsPlayingAudio(true);
+
+        // Decode base64 audio
+        const binaryString = atob(data.audio);
+        const bytes = new Uint8Array(binaryString.length);
+        for (let i = 0; i < binaryString.length; i++) {
+          bytes[i] = binaryString.charCodeAt(i);
+        }
+
+        // Create blob with correct mime type
+        const mimeType = data.format === 'mp3' ? 'audio/mpeg' : `audio/${data.format}`;
+        const audioBlob = new Blob([bytes], { type: mimeType });
+
+        // Play audio
+        await audioPlayerRef.current.play(audioBlob);
+
+        setIsPlayingAudio(false);
+        addLog('success', 'Audio playback complete');
+      } catch (error: any) {
+        console.error('[App] Audio playback error:', error);
+        addLog('error', `Audio playback failed: ${error.message}`);
+        setIsPlayingAudio(false);
+      }
+    });
+
+    return () => {
+      unsubStatus();
+      unsubPong();
+      unsubActivity();
+      unsubChunk();
+      unsubTranscription();
+      unsubAudioOutput();
+    };
+  }, [ws]);
+
+  useEffect(() => {
+    if (ws.isConnected) {
+      addLog('success', 'WebSocket connected');
+    } else {
+      addLog('system', 'WebSocket disconnected');
+    }
+  }, [ws.isConnected]);
+
+  // Auto-scroll to bottom when new logs are added
+  useEffect(() => {
+    logEndRef.current?.scrollIntoView({ behavior: 'smooth' });
+  }, [logs, currentAssistantMessage]);
+
+  const handlePing = () => {
+    ws.sendPing();
+    addLog('info', 'Sent ping to server');
+  };
+
+  const handleSendMessage = (e: React.FormEvent) => {
+    e.preventDefault();
+    if (!userInput.trim() || !ws.isConnected) return;
+
+    // Send message to server
+    ws.sendUserMessage(userInput);
+
+    // Clear input and reset streaming state
+    setUserInput('');
+    setCurrentAssistantMessage('');
+  };
+
+  const handleAudioRecorded = async (audioBlob: Blob, format: string) => {
+    if (!ws.isConnected) {
+      addLog('error', 'Cannot send audio - not connected to server');
+      return;
+    }
+
+    try {
+      setIsProcessingAudio(true);
+      addLog('info', 'Sending audio to server...');
+
+      // Convert blob to base64
+      const arrayBuffer = await audioBlob.arrayBuffer();
+      const base64Audio = btoa(
+        new Uint8Array(arrayBuffer).reduce((data, byte) => data + String.fromCharCode(byte), '')
+      );
+
+      // Send audio chunk via WebSocket
+      ws.send({
+        type: 'audio_chunk',
+        payload: {
+          audio: base64Audio,
+          format: format,
+          isLast: true,
+        },
+      });
+
+      console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`);
+    } catch (error: any) {
+      console.error('[App] Error sending audio:', error);
+      addLog('error', `Failed to send audio: ${error.message}`);
+      setIsProcessingAudio(false);
+    }
+  };
+
+  return (
+    <div className="app">
+      <header className="header">
+        <h1>Voice Assistant</h1>
+        <div className={`status-indicator ${ws.isConnected ? 'connected' : 'disconnected'}`}>
+          {ws.isConnected ? 'connected' : 'disconnected'}
+        </div>
+      </header>
+
+      <main className="main">
+        <div className="chat-interface">
+          <h2>Chat with Assistant</h2>
+
+          <VoiceControls
+            onAudioRecorded={handleAudioRecorded}
+            isProcessing={isProcessingAudio}
+            isPlaying={isPlayingAudio}
+          />
+
+          <div className="activity-log">
+            <div className="log-entries">
+              {logs.map((log) => (
+                <div key={log.id} className={`log-entry ${log.type}`}>
+                  <span className="log-time">{new Date(log.timestamp).toLocaleTimeString()}</span>
+                  <span className="log-message">{log.message}</span>
+                  {log.metadata && (
+                    <details className="log-metadata">
+                      <summary>Details</summary>
+                      <pre>{JSON.stringify(log.metadata, null, 2)}</pre>
+                    </details>
+                  )}
+                </div>
+              ))}
+              {currentAssistantMessage && (
+                <div className="log-entry assistant streaming">
+                  <span className="log-time">{new Date().toLocaleTimeString()}</span>
+                  <span className="log-message">{currentAssistantMessage}</span>
+                  <span className="streaming-indicator">...</span>
+                </div>
+              )}
+              <div ref={logEndRef} />
+            </div>
+          </div>
+
+          <form onSubmit={handleSendMessage} className="message-form">
+            <input
+              type="text"
+              value={userInput}
+              onChange={(e) => setUserInput(e.target.value)}
+              placeholder="Type a message to the assistant..."
+              disabled={!ws.isConnected}
+              className="message-input"
+            />
+            <button type="submit" disabled={!ws.isConnected || !userInput.trim()} className="send-button">
+              Send
+            </button>
+          </form>
+
+          <div className="test-controls">
+            <button onClick={handlePing} disabled={!ws.isConnected} className="ping-button">
+              Send Ping (Test)
+            </button>
+          </div>
+        </div>
+      </main>
+    </div>
+  );
+}
+
+export default App;

packages/voice-assistant/src/ui/components/VoiceControls.tsx

@@ -0,0 +1,113 @@
+import { useState, useEffect, useRef } from 'react';
+import { createAudioRecorder, type AudioRecorder } from '../lib/audio-capture';
+
+interface VoiceControlsProps {
+  onAudioRecorded: (audio: Blob, format: string) => void;
+  isProcessing: boolean;
+  isPlaying?: boolean;
+}
+
+export function VoiceControls({ onAudioRecorded, isProcessing, isPlaying }: VoiceControlsProps) {
+  const [isRecording, setIsRecording] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [permissionState, setPermissionState] = useState<'prompt' | 'granted' | 'denied'>('prompt');
+  const recorderRef = useRef<AudioRecorder | null>(null);
+
+  useEffect(() => {
+    // Initialize recorder
+    recorderRef.current = createAudioRecorder();
+
+    // Check microphone permission
+    if (navigator.permissions) {
+      navigator.permissions
+        .query({ name: 'microphone' as PermissionName })
+        .then((result) => {
+          setPermissionState(result.state);
+          result.onchange = () => {
+            setPermissionState(result.state);
+          };
+        })
+        .catch(() => {
+          setPermissionState('prompt');
+        });
+    }
+  }, []);
+
+  async function handleToggleRecording() {
+    const recorder = recorderRef.current;
+    if (!recorder) return;
+
+    try {
+      setError(null);
+
+      if (isRecording) {
+        // Stop recording
+        console.log('[VoiceControls] Stopping recording...');
+        const audioBlob = await recorder.stop();
+        setIsRecording(false);
+
+        // Get format from blob type
+        const format = audioBlob.type || 'audio/webm';
+        console.log(`[VoiceControls] Recording complete: ${audioBlob.size} bytes, format: ${format}`);
+
+        // Send to parent
+        onAudioRecorded(audioBlob, format);
+      } else {
+        // Start recording
+        console.log('[VoiceControls] Starting recording...');
+        await recorder.start();
+        setIsRecording(true);
+        setPermissionState('granted');
+      }
+    } catch (err: any) {
+      console.error('[VoiceControls] Recording error:', err);
+      setError(err.message || 'Failed to access microphone');
+      setIsRecording(false);
+
+      if (err.message.includes('Permission denied')) {
+        setPermissionState('denied');
+      }
+    }
+  }
+
+  const canRecord = !isProcessing && !isPlaying;
+  const buttonDisabled = !canRecord || permissionState === 'denied';
+
+  return (
+    <div className="voice-controls">
+      <button
+        onClick={handleToggleRecording}
+        disabled={buttonDisabled}
+        className={`voice-button ${isRecording ? 'recording' : ''} ${isProcessing ? 'processing' : ''}`}
+        title={isRecording ? 'Stop recording' : 'Start recording'}
+      >
+        {isRecording ? (
+          <>
+            <span className="recording-indicator" />
+            <span>Recording...</span>
+          </>
+        ) : isProcessing ? (
+          <>
+            <span className="processing-indicator" />
+            <span>Processing...</span>
+          </>
+        ) : (
+          <>
+            <span className="microphone-icon">🎤</span>
+            <span>Record</span>
+          </>
+        )}
+      </button>
+
+      {error && <div className="voice-error">{error}</div>}
+
+      {permissionState === 'denied' && (
+        <div className="permission-denied">
+          Microphone access denied. Please enable it in your browser settings.
+        </div>
+      )}
+
+      {isPlaying && <div className="playing-indicator">🔊 Playing response...</div>}
+    </div>
+  );
+}

packages/voice-assistant/src/ui/hooks/useWebSocket.ts

@@ -0,0 +1,136 @@
+import { useEffect, useRef, useState, useCallback } from 'react';
+
+interface WebSocketMessage {
+  type: string;
+  payload: unknown;
+}
+
+type MessageHandler = (payload: unknown) => void;
+
+export interface UseWebSocketReturn {
+  isConnected: boolean;
+  send: (message: WebSocketMessage) => void;
+  on: (type: string, handler: MessageHandler) => () => void;
+  sendPing: () => void;
+  sendUserMessage: (message: string) => void;
+}
+
+export function useWebSocket(url: string): UseWebSocketReturn {
+  const [isConnected, setIsConnected] = useState(false);
+  const wsRef = useRef<WebSocket | null>(null);
+  const handlersRef = useRef<Map<string, Set<MessageHandler>>>(new Map());
+  const reconnectTimeoutRef = useRef<number>();
+
+  const connect = useCallback(() => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      return;
+    }
+
+    try {
+      const ws = new WebSocket(url);
+
+      ws.onopen = () => {
+        console.log('[WS] Connected to server');
+        setIsConnected(true);
+      };
+
+      ws.onclose = () => {
+        console.log('[WS] Disconnected from server');
+        setIsConnected(false);
+
+        // Attempt to reconnect after 3 seconds
+        reconnectTimeoutRef.current = window.setTimeout(() => {
+          console.log('[WS] Attempting to reconnect...');
+          connect();
+        }, 3000);
+      };
+
+      ws.onerror = (error) => {
+        console.error('[WS] Error:', error);
+      };
+
+      ws.onmessage = (event) => {
+        try {
+          const message = JSON.parse(event.data) as WebSocketMessage;
+          console.log(`[WS] Received message type: ${message.type}`, message.payload);
+
+          // Call all registered handlers for this message type
+          const handlers = handlersRef.current.get(message.type);
+          if (handlers) {
+            handlers.forEach((handler) => {
+              try {
+                handler(message.payload);
+              } catch (err) {
+                console.error(`[WS] Error in handler for ${message.type}:`, err);
+              }
+            });
+          }
+        } catch (err) {
+          console.error('[WS] Failed to parse message:', err);
+        }
+      };
+
+      wsRef.current = ws;
+    } catch (err) {
+      console.error('[WS] Failed to create WebSocket:', err);
+    }
+  }, [url]);
+
+  useEffect(() => {
+    connect();
+
+    return () => {
+      if (reconnectTimeoutRef.current) {
+        clearTimeout(reconnectTimeoutRef.current);
+      }
+      if (wsRef.current) {
+        wsRef.current.close();
+      }
+    };
+  }, [connect]);
+
+  const send = useCallback((message: WebSocketMessage) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify(message));
+    } else {
+      console.warn('[WS] Cannot send message - not connected');
+    }
+  }, []);
+
+  const on = useCallback((type: string, handler: MessageHandler) => {
+    if (!handlersRef.current.has(type)) {
+      handlersRef.current.set(type, new Set());
+    }
+    handlersRef.current.get(type)!.add(handler);
+
+    // Return cleanup function
+    return () => {
+      const handlers = handlersRef.current.get(type);
+      if (handlers) {
+        handlers.delete(handler);
+        if (handlers.size === 0) {
+          handlersRef.current.delete(type);
+        }
+      }
+    };
+  }, []);
+
+  const sendPing = useCallback(() => {
+    send({ type: 'ping', payload: {} });
+  }, [send]);
+
+  const sendUserMessage = useCallback(
+    (message: string) => {
+      send({ type: 'user_message', payload: { message } });
+    },
+    [send]
+  );
+
+  return {
+    isConnected,
+    send,
+    on,
+    sendPing,
+    sendUserMessage,
+  };
+}

packages/voice-assistant/src/ui/index.css

@@ -0,0 +1,31 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+* {
+  box-sizing: border-box;
+  margin: 0;
+  padding: 0;
+}
+
+body {
+  margin: 0;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+#root {
+  width: 100%;
+  min-height: 100vh;
+}

packages/voice-assistant/src/ui/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Voice Assistant</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/main.tsx"></script>
+  </body>
+</html>

packages/voice-assistant/src/ui/lib/audio-capture.ts

@@ -0,0 +1,184 @@
+export interface AudioCaptureConfig {
+  mimeType?: string;
+  audioBitsPerSecond?: number;
+}
+
+export interface AudioRecorder {
+  start(): Promise<void>;
+  stop(): Promise<Blob>;
+  isRecording(): boolean;
+  getSupportedMimeType(): string | null;
+}
+
+export function createAudioRecorder(config?: AudioCaptureConfig): AudioRecorder {
+  let mediaRecorder: MediaRecorder | null = null;
+  let audioChunks: Blob[] = [];
+  let stream: MediaStream | null = null;
+  let recording = false;
+
+  // Detect supported mime type
+  function getSupportedMimeType(): string | null {
+    const types = [
+      'audio/webm;codecs=opus',
+      'audio/webm',
+      'audio/ogg;codecs=opus',
+      'audio/ogg',
+      'audio/mp4',
+      'audio/wav',
+    ];
+
+    for (const type of types) {
+      if (MediaRecorder.isTypeSupported(type)) {
+        return type;
+      }
+    }
+
+    return null;
+  }
+
+  async function start(): Promise<void> {
+    if (recording) {
+      throw new Error('Already recording');
+    }
+
+    try {
+      // Request microphone permission with WebRTC audio processing constraints
+      stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          echoCancellation: true,      // Remove echo from speakers
+          noiseSuppression: true,       // Remove background noise
+          autoGainControl: true,        // Normalize volume
+          sampleRate: 16000,            // Optimal for speech (Whisper works well with this)
+          channelCount: 1,              // Mono audio (sufficient for voice)
+        },
+        video: false,
+      });
+
+      // Check which constraints were actually applied
+      const track = stream.getAudioTracks()[0];
+      const settings = track.getSettings();
+      console.log('[AudioRecorder] Audio settings:', settings);
+
+      // Try WebM/Opus first (best compression and quality for speech)
+      const mimeType = config?.mimeType ||
+        (MediaRecorder.isTypeSupported('audio/webm;codecs=opus')
+          ? 'audio/webm;codecs=opus'
+          : getSupportedMimeType());
+
+      if (!mimeType) {
+        throw new Error('No supported audio mime type found');
+      }
+
+      // Create MediaRecorder
+      const options: MediaRecorderOptions = {
+        mimeType,
+      };
+
+      if (config?.audioBitsPerSecond) {
+        options.audioBitsPerSecond = config.audioBitsPerSecond;
+      }
+
+      mediaRecorder = new MediaRecorder(stream, options);
+      audioChunks = [];
+
+      // Collect data chunks
+      mediaRecorder.ondataavailable = (event) => {
+        if (event.data.size > 0) {
+          audioChunks.push(event.data);
+          console.log(`[AudioRecorder] Chunk received: ${event.data.size} bytes`);
+        }
+      };
+
+      // Start recording
+      mediaRecorder.start(100); // Collect chunks every 100ms
+      recording = true;
+
+      console.log('[AudioRecorder] Recording started with constraints:', {
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+        sampleRate: 16000,
+        channelCount: 1,
+        mimeType,
+      });
+    } catch (error: any) {
+      console.error('[AudioRecorder] Failed to start recording:', error);
+      if (stream) {
+        stream.getTracks().forEach((track) => track.stop());
+        stream = null;
+      }
+      throw new Error(`Failed to start audio recording: ${error.message}`);
+    }
+  }
+
+  async function stop(): Promise<Blob> {
+    if (!recording || !mediaRecorder) {
+      throw new Error('Not recording');
+    }
+
+    return new Promise((resolve, reject) => {
+      if (!mediaRecorder) {
+        reject(new Error('MediaRecorder not initialized'));
+        return;
+      }
+
+      mediaRecorder.onstop = () => {
+        // Create blob from chunks
+        const mimeType = mediaRecorder?.mimeType || 'audio/webm';
+        const audioBlob = new Blob(audioChunks, { type: mimeType });
+
+        console.log('[AudioRecorder] Recording stopped:', {
+          size: audioBlob.size,
+          type: mimeType,
+          chunks: audioChunks.length,
+        });
+
+        // Clean up
+        if (stream) {
+          stream.getTracks().forEach((track) => {
+            track.stop();
+            console.log('[AudioRecorder] Stopped track:', track.label);
+          });
+          stream = null;
+        }
+
+        recording = false;
+        mediaRecorder = null;
+        audioChunks = [];
+
+        resolve(audioBlob);
+      };
+
+      mediaRecorder.onerror = (event: Event) => {
+        console.error('[AudioRecorder] MediaRecorder error:', event);
+        reject(new Error('MediaRecorder error during stop'));
+      };
+
+      // Stop recording
+      mediaRecorder.stop();
+    });
+  }
+
+  function isRecordingFunc(): boolean {
+    return recording;
+  }
+
+  return {
+    start,
+    stop,
+    isRecording: isRecordingFunc,
+    getSupportedMimeType,
+  };
+}
+
+export function checkMicrophonePermission(): Promise<PermissionState> {
+  // Check if browser supports permissions API
+  if (!navigator.permissions) {
+    return Promise.resolve('prompt' as PermissionState);
+  }
+
+  return navigator.permissions
+    .query({ name: 'microphone' as PermissionName })
+    .then((result) => result.state)
+    .catch(() => 'prompt' as PermissionState);
+}

packages/voice-assistant/src/ui/lib/audio-playback.ts

@@ -0,0 +1,116 @@
+export interface AudioPlayer {
+  play(audioData: Blob): Promise<void>;
+  stop(): void;
+  isPlaying(): boolean;
+  clearQueue(): void;
+}
+
+export function createAudioPlayer(): AudioPlayer {
+  let currentAudio: HTMLAudioElement | null = null;
+  let playing = false;
+  let queue: Blob[] = [];
+  let isProcessingQueue = false;
+
+  async function play(audioData: Blob): Promise<void> {
+    // Add to queue
+    queue.push(audioData);
+
+    // Start processing queue if not already processing
+    if (!isProcessingQueue) {
+      processQueue();
+    }
+  }
+
+  async function processQueue(): Promise<void> {
+    if (isProcessingQueue || queue.length === 0) {
+      return;
+    }
+
+    isProcessingQueue = true;
+
+    while (queue.length > 0) {
+      const audioData = queue.shift()!;
+      await playAudio(audioData);
+    }
+
+    isProcessingQueue = false;
+  }
+
+  async function playAudio(audioData: Blob): Promise<void> {
+    return new Promise((resolve, reject) => {
+      try {
+        // Create blob URL
+        const audioUrl = URL.createObjectURL(audioData);
+
+        // Create audio element
+        const audio = new Audio(audioUrl);
+        currentAudio = audio;
+        playing = true;
+
+        console.log(`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`);
+
+        audio.onended = () => {
+          console.log('[AudioPlayer] Playback finished');
+          playing = false;
+          currentAudio = null;
+
+          // Clean up blob URL
+          URL.revokeObjectURL(audioUrl);
+
+          resolve();
+        };
+
+        audio.onerror = (error) => {
+          console.error('[AudioPlayer] Playback error:', error);
+          playing = false;
+          currentAudio = null;
+
+          // Clean up blob URL
+          URL.revokeObjectURL(audioUrl);
+
+          reject(new Error('Audio playback failed'));
+        };
+
+        // Start playback
+        audio.play().catch((error) => {
+          console.error('[AudioPlayer] Failed to start playback:', error);
+          playing = false;
+          currentAudio = null;
+          URL.revokeObjectURL(audioUrl);
+          reject(error);
+        });
+      } catch (error) {
+        console.error('[AudioPlayer] Error creating audio element:', error);
+        playing = false;
+        currentAudio = null;
+        reject(error);
+      }
+    });
+  }
+
+  function stop(): void {
+    if (currentAudio) {
+      currentAudio.pause();
+      currentAudio.currentTime = 0;
+      currentAudio = null;
+    }
+    playing = false;
+    queue = [];
+    isProcessingQueue = false;
+  }
+
+  function isPlayingFunc(): boolean {
+    return playing;
+  }
+
+  function clearQueue(): void {
+    queue = [];
+  }
+
+  return {
+    play,
+    stop,
+    isPlaying: isPlayingFunc,
+    clearQueue,
+  };
+}

packages/voice-assistant/src/ui/main.tsx

@@ -0,0 +1,10 @@
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import App from './App.tsx'
+import './index.css'
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>,
+)

packages/voice-assistant/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />