Merge pull request #22 from boudra/local-streaming-speech-tts

voice: harden local voice agent path and UUID validation

packages/server/src/client/daemon-client.ts

@@ -12,9 +12,7 @@ import type {
   AgentStreamEventPayload,
   AgentSnapshotPayload,
   AgentPermissionResolvedMessage,
-  VoiceConversationLoadedMessage,
   CreateAgentRequestMessage,
-  DeleteVoiceConversationResponseMessage,
   FileDownloadTokenResponse,
   FileExplorerResponse,
   GitDiffResponse,
@@ -34,7 +32,6 @@ import type {
   ProjectIconResponse,
   ListCommandsResponse,
   ExecuteCommandResponse,
-  ListVoiceConversationsResponseMessage,
   ListProviderModelsResponseMessage,
   SpeechModelsListResponse,
   SpeechModelsDownloadResponse,
@@ -184,9 +181,6 @@ export type CreateAgentRequestOptions = {
   labels?: Record<string, string>;
 } & AgentConfigOverrides;
 
-type VoiceConversationLoadedPayload = VoiceConversationLoadedMessage["payload"];
-type ListVoiceConversationsPayload = ListVoiceConversationsResponseMessage["payload"];
-type DeleteVoiceConversationPayload = DeleteVoiceConversationResponseMessage["payload"];
 type GitDiffPayload = GitDiffResponse["payload"];
 type HighlightedDiffPayload = HighlightedDiffResponse["payload"];
 type CheckoutStatusPayload = CheckoutStatusResponse["payload"];
@@ -765,10 +759,6 @@ export class DaemonClient {
     }
   }
 
-  sendUserMessage(text: string): void {
-    this.sendSessionMessage({ type: "user_text", text });
-  }
-
   clearAgentAttention(agentId: string | string[]): void {
     this.sendSessionMessage({ type: "clear_agent_attention", agentId });
   }
@@ -926,87 +916,6 @@ export class DaemonClient {
     }
   }
 
-  // ============================================================================
-  // Voice Conversation RPC
-  // ============================================================================
-
-  async loadVoiceConversation(
-    voiceConversationId: string,
-    requestId?: string
-  ): Promise<VoiceConversationLoadedPayload> {
-    const resolvedRequestId = this.createRequestId(requestId);
-    const message = SessionInboundMessageSchema.parse({
-      type: "load_voice_conversation_request",
-      voiceConversationId,
-      requestId: resolvedRequestId,
-    });
-    return this.sendRequest({
-      requestId: resolvedRequestId,
-      message,
-      timeout: 10000,
-      options: { skipQueue: true },
-      select: (msg) => {
-        if (msg.type !== "voice_conversation_loaded") {
-          return null;
-        }
-        if (msg.payload.requestId !== resolvedRequestId) {
-          return null;
-        }
-        return msg.payload;
-      },
-    });
-  }
-
-  async listVoiceConversations(requestId?: string): Promise<ListVoiceConversationsPayload> {
-    const resolvedRequestId = this.createRequestId(requestId);
-    const message = SessionInboundMessageSchema.parse({
-      type: "list_voice_conversations_request",
-      requestId: resolvedRequestId,
-    });
-    return this.sendRequest({
-      requestId: resolvedRequestId,
-      message,
-      timeout: 10000,
-      options: { skipQueue: true },
-      select: (msg) => {
-        if (msg.type !== "list_voice_conversations_response") {
-          return null;
-        }
-        if (msg.payload.requestId !== resolvedRequestId) {
-          return null;
-        }
-        return msg.payload;
-      },
-    });
-  }
-
-  async deleteVoiceConversation(
-    voiceConversationId: string,
-    requestId?: string
-  ): Promise<DeleteVoiceConversationPayload> {
-    const resolvedRequestId = this.createRequestId(requestId);
-    const message = SessionInboundMessageSchema.parse({
-      type: "delete_voice_conversation_request",
-      voiceConversationId,
-      requestId: resolvedRequestId,
-    });
-    return this.sendRequest({
-      requestId: resolvedRequestId,
-      message,
-      timeout: 10000,
-      options: { skipQueue: true },
-      select: (msg) => {
-        if (msg.type !== "delete_voice_conversation_response") {
-          return null;
-        }
-        if (msg.payload.requestId !== resolvedRequestId) {
-          return null;
-        }
-        return msg.payload;
-      },
-    });
-  }
-
   // ============================================================================
   // Agent Lifecycle
   // ============================================================================
@@ -1376,8 +1285,8 @@ export class DaemonClient {
   // Audio / Voice
   // ============================================================================
 
-  async setVoiceConversation(enabled: boolean, voiceConversationId?: string): Promise<void> {
-    this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId });
+  async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise<void> {
+    this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId });
   }
 
   async sendVoiceAudioChunk(

packages/server/src/server/agent/activity-curator.test.ts

@@ -66,6 +66,22 @@ describe("curateAgentActivity", () => {
       expect(result).toBe("[ListFiles]");
     });
 
+    test("does not treat generic double-underscore tool names as MCP calls", () => {
+      const timeline: AgentTimelineItem[] = [
+        {
+          type: "tool_call",
+          callId: "call-1",
+          name: "custom__tool",
+          input: {},
+          status: "completed",
+        },
+      ];
+
+      const result = curateAgentActivity(timeline);
+
+      expect(result).toBe("[custom__tool]");
+    });
+
     test("serializes todo items as [Tasks]", () => {
       const timeline: AgentTimelineItem[] = [
         {
@@ -328,5 +344,37 @@ describe("curateAgentActivity", () => {
 
       expect(result).toBe("[Grep] TODO");
     });
+
+    test("shows speak tool text input", () => {
+      const timeline: AgentTimelineItem[] = [
+        {
+          type: "tool_call",
+          callId: "s1",
+          name: "speak",
+          input: { text: "hello from voice" },
+          status: "completed",
+        },
+      ];
+
+      const result = curateAgentActivity(timeline);
+      expect(result).toBe('[speak] {"text":"hello from voice"}');
+    });
+
+    test("shows MCP tool input JSON", () => {
+      const timeline: AgentTimelineItem[] = [
+        {
+          type: "tool_call",
+          callId: "m1",
+          name: "paseo__create_agent",
+          input: { cwd: "/tmp/repo", initialPrompt: "do the thing" },
+          status: "completed",
+        },
+      ];
+
+      const result = curateAgentActivity(timeline);
+      expect(result).toBe(
+        '[paseo__create_agent] {"cwd":"/tmp/repo","initialPrompt":"do the thing"}'
+      );
+    });
   });
 });

packages/server/src/server/agent/activity-curator.ts

@@ -2,6 +2,7 @@ import type { AgentTimelineItem } from "./agent-sdk-types.js";
 import { extractPrincipalParam } from "../../utils/tool-call-parsers.js";
 
 const DEFAULT_MAX_ITEMS = 40;
+const MAX_TOOL_INPUT_CHARS = 400;
 
 function appendText(buffer: string, text: string): string {
   const normalized = text.trim();
@@ -25,6 +26,35 @@ function flushBuffers(lines: string[], buffers: { message: string; thought: stri
   buffers.thought = "";
 }
 
+function isLikelyMcpToolCall(name: string): boolean {
+  const normalized = name.toLowerCase();
+  return (
+    normalized === "speak" ||
+    normalized.startsWith("mcp") ||
+    normalized.includes("mcp__") ||
+    normalized.startsWith("paseo") ||
+    normalized.includes("paseo__")
+  );
+}
+
+function formatToolInputJson(input: unknown): string | null {
+  if (input === undefined) {
+    return null;
+  }
+  try {
+    const encoded = JSON.stringify(input);
+    if (!encoded) {
+      return null;
+    }
+    if (encoded.length <= MAX_TOOL_INPUT_CHARS) {
+      return encoded;
+    }
+    return `${encoded.slice(0, MAX_TOOL_INPUT_CHARS)}...`;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Collapse timeline items:
  * - Dedupe tool calls by callId (pending/completed -> single)
@@ -127,6 +157,11 @@ export function curateAgentActivity(
         break;
       case "tool_call": {
         flushBuffers(lines, buffers);
+        const inputJson = formatToolInputJson(item.input);
+        if (isLikelyMcpToolCall(item.name) && inputJson) {
+          lines.push(`[${item.name}] ${inputJson}`);
+          break;
+        }
         const principal = extractPrincipalParam(item.input);
         if (principal) {
           lines.push(`[${item.name}] ${principal}`);

packages/server/src/server/agent/agent-management-mcp.ts

@@ -23,8 +23,6 @@
 
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
-import { homedir } from "node:os";
-import { resolve } from "node:path";
 import { ensureValidJson } from "../json-utils.js";
 import type { Logger } from "pino";
 
@@ -51,6 +49,7 @@ import { AgentStorage } from "./agent-storage.js";
 import { createWorktree } from "../../utils/worktree.js";
 import { WaitForAgentTracker } from "./wait-for-agent-tracker.js";
 import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js";
+import { expandUserPath } from "../path-utils.js";
 
 export interface AgentManagementMcpOptions {
   agentManager: AgentManager;
@@ -77,13 +76,6 @@ const AgentStatusEnum = z.enum([
 // 50 seconds - surface friendly message before SDK tool timeout (~60s)
 const AGENT_WAIT_TIMEOUT_MS = 50000;
 
-function expandPath(path: string): string {
-  if (path.startsWith("~/") || path === "~") {
-    return resolve(homedir(), path.slice(2));
-  }
-  return resolve(path);
-}
-
 async function waitForAgentWithTimeout(
   agentManager: AgentManager,
   agentId: string,
@@ -320,7 +312,7 @@ export async function createAgentManagementMcpServer(
         title: string;
       };
 
-      let resolvedCwd = expandPath(cwd);
+      let resolvedCwd = expandUserPath(cwd);
 
       if (worktreeName) {
         if (!baseBranch) {

packages/server/src/server/agent/agent-manager.test.ts

@@ -118,7 +118,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => "agent-without-model",
+      idFactory: () => "00000000-0000-4000-8000-000000000101",
     });
 
     const snapshot = await manager.createAgent({
@@ -129,7 +129,7 @@ describe("AgentManager", () => {
     expect(snapshot.model).toBeUndefined();
   });
 
-  test("createAgent persists provided title before returning", async () => {
+  test("createAgent fails when cwd does not exist", async () => {
     const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
     const storagePath = join(workdir, "agents");
     const storage = new AgentStorage(storagePath, logger);
@@ -139,7 +139,72 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => "agent-with-title",
+    });
+
+    await expect(
+      manager.createAgent({
+        provider: "codex",
+        cwd: join(workdir, "does-not-exist"),
+      })
+    ).rejects.toThrow("Working directory does not exist");
+  });
+
+  test("createAgent fails when generated agent ID is not a UUID", async () => {
+    const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
+    const storagePath = join(workdir, "agents");
+    const storage = new AgentStorage(storagePath, logger);
+    const manager = new AgentManager({
+      clients: {
+        codex: new TestAgentClient(),
+      },
+      registry: storage,
+      logger,
+      idFactory: () => "not-a-uuid",
+    });
+
+    await expect(
+      manager.createAgent({
+        provider: "codex",
+        cwd: workdir,
+      })
+    ).rejects.toThrow("createAgent: agentId must be a UUID");
+  });
+
+  test("createAgent fails when explicit agent ID is not a UUID", async () => {
+    const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
+    const storagePath = join(workdir, "agents");
+    const storage = new AgentStorage(storagePath, logger);
+    const manager = new AgentManager({
+      clients: {
+        codex: new TestAgentClient(),
+      },
+      registry: storage,
+      logger,
+    });
+
+    await expect(
+      manager.createAgent(
+        {
+          provider: "codex",
+          cwd: workdir,
+        },
+        "not-a-uuid"
+      )
+    ).rejects.toThrow("createAgent: agentId must be a UUID");
+  });
+
+  test("createAgent persists provided title before returning", async () => {
+    const agentId = "00000000-0000-4000-8000-000000000102";
+    const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
+    const storagePath = join(workdir, "agents");
+    const storage = new AgentStorage(storagePath, logger);
+    const manager = new AgentManager({
+      clients: {
+        codex: new TestAgentClient(),
+      },
+      registry: storage,
+      logger,
+      idFactory: () => agentId,
     });
 
     const snapshot = await manager.createAgent({
@@ -148,12 +213,12 @@ describe("AgentManager", () => {
       title: "Fix Login Bug",
     });
 
-    expect(snapshot.id).toBe("agent-with-title");
+    expect(snapshot.id).toBe(agentId);
     expect(snapshot.lifecycle).toBe("idle");
 
-    const persisted = await storage.get("agent-with-title");
+    const persisted = await storage.get(agentId);
     expect(persisted?.title).toBe("Fix Login Bug");
-    expect(persisted?.id).toBe("agent-with-title");
+    expect(persisted?.id).toBe(agentId);
   });
 
   test("createAgent populates runtimeInfo after session creation", async () => {
@@ -166,7 +231,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => "agent-with-runtime-info",
+      idFactory: () => "00000000-0000-4000-8000-000000000103",
     });
 
     const snapshot = await manager.createAgent({
@@ -191,7 +256,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => "agent-with-run-runtime",
+      idFactory: () => "00000000-0000-4000-8000-000000000104",
     });
 
     const snapshot = await manager.createAgent({
@@ -211,6 +276,10 @@ describe("AgentManager", () => {
     const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
     const storagePath = join(workdir, "agents");
     const storage = new AgentStorage(storagePath, logger);
+    const generatedAgentIds = [
+      "00000000-0000-4000-8000-000000000105",
+      "00000000-0000-4000-8000-000000000106",
+    ];
     let agentCounter = 0;
     const manager = new AgentManager({
       clients: {
@@ -218,7 +287,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => `agent-${agentCounter++}`,
+      idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(),
     });
 
     // Create a normal agent
@@ -242,6 +311,7 @@ describe("AgentManager", () => {
   });
 
   test("getAgent returns internal agents by ID", async () => {
+    const internalAgentId = "00000000-0000-4000-8000-000000000107";
     const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
     const storagePath = join(workdir, "agents");
     const storage = new AgentStorage(storagePath, logger);
@@ -251,7 +321,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => "internal-agent",
+      idFactory: () => internalAgentId,
     });
 
     await manager.createAgent({
@@ -261,7 +331,7 @@ describe("AgentManager", () => {
       internal: true,
     });
 
-    const agent = manager.getAgent("internal-agent");
+    const agent = manager.getAgent(internalAgentId);
     expect(agent).not.toBeNull();
     expect(agent?.internal).toBe(true);
   });
@@ -270,6 +340,10 @@ describe("AgentManager", () => {
     const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
     const storagePath = join(workdir, "agents");
     const storage = new AgentStorage(storagePath, logger);
+    const generatedAgentIds = [
+      "00000000-0000-4000-8000-000000000108",
+      "00000000-0000-4000-8000-000000000109",
+    ];
     let agentCounter = 0;
     const manager = new AgentManager({
       clients: {
@@ -277,7 +351,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => `agent-${agentCounter++}`,
+      idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(),
     });
 
     const receivedEvents: string[] = [];
@@ -303,11 +377,12 @@ describe("AgentManager", () => {
     });
 
     // Should only have events from the normal agent
-    expect(receivedEvents.filter((id) => id === "agent-0").length).toBeGreaterThan(0);
-    expect(receivedEvents.filter((id) => id === "agent-1").length).toBe(0);
+    expect(receivedEvents.filter((id) => id === generatedAgentIds[0]).length).toBeGreaterThan(0);
+    expect(receivedEvents.filter((id) => id === generatedAgentIds[1]).length).toBe(0);
   });
 
   test("subscribe emits state events for internal agents when subscribed by agentId", async () => {
+    const internalAgentId = "00000000-0000-4000-8000-000000000110";
     const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
     const storagePath = join(workdir, "agents");
     const storage = new AgentStorage(storagePath, logger);
@@ -317,7 +392,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => "internal-agent",
+      idFactory: () => internalAgentId,
     });
 
     const receivedEvents: string[] = [];
@@ -328,7 +403,7 @@ describe("AgentManager", () => {
           receivedEvents.push(event.agent.id);
         }
       },
-      { agentId: "internal-agent", replayState: false }
+      { agentId: internalAgentId, replayState: false }
     );
 
     await manager.createAgent({
@@ -339,10 +414,26 @@ describe("AgentManager", () => {
     });
 
     // Should receive events when subscribed by specific agentId
-    expect(receivedEvents.filter((id) => id === "internal-agent").length).toBeGreaterThan(0);
+    expect(receivedEvents.filter((id) => id === internalAgentId).length).toBeGreaterThan(0);
+  });
+
+  test("subscribe fails when filter agentId is not a UUID", () => {
+    const manager = new AgentManager({
+      clients: {
+        codex: new TestAgentClient(),
+      },
+      logger,
+    });
+
+    expect(() =>
+      manager.subscribe(() => {}, {
+        agentId: "invalid-agent-id",
+      })
+    ).toThrow("subscribe: agentId must be a UUID");
   });
 
   test("onAgentAttention is not called for internal agents", async () => {
+    const internalAgentId = "00000000-0000-4000-8000-000000000111";
     const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
     const storagePath = join(workdir, "agents");
     const storage = new AgentStorage(storagePath, logger);
@@ -353,7 +444,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => "internal-agent",
+      idFactory: () => internalAgentId,
       onAgentAttention: ({ agentId }) => {
         attentionCalls.push(agentId);
       },
@@ -461,7 +552,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => "plan-mode-agent",
+      idFactory: () => "00000000-0000-4000-8000-000000000112",
     });
 
     // Create agent in plan mode
@@ -588,7 +679,7 @@ describe("AgentManager", () => {
       },
       registry: storage,
       logger,
-      idFactory: () => "close-race-agent",
+      idFactory: () => "00000000-0000-4000-8000-000000000113",
     });
 
     const snapshot = await manager.createAgent({

packages/server/src/server/agent/agent-manager.ts

@@ -1,10 +1,12 @@
 import { randomUUID } from "node:crypto";
 import { resolve } from "node:path";
+import { stat } from "node:fs/promises";
 import {
   AGENT_LIFECYCLE_STATUSES,
   type AgentLifecycleStatus,
 } from "../../shared/agent-lifecycle.js";
 import type { Logger } from "pino";
+import { z } from "zod";
 
 import type {
   AgentCapabilityFlags,
@@ -182,6 +184,7 @@ const BUSY_STATUSES: AgentLifecycleStatus[] = [
   "initializing",
   "running",
 ];
+const AgentIdSchema = z.string().uuid();
 
 function isAgentBusy(status: AgentLifecycleStatus): boolean {
   return BUSY_STATUSES.includes(status);
@@ -201,6 +204,14 @@ function createAbortError(
   return Object.assign(new Error(message), { name: "AbortError" });
 }
 
+function validateAgentId(agentId: string, source: string): string {
+  const result = AgentIdSchema.safeParse(agentId);
+  if (!result.success) {
+    throw new Error(`${source}: agentId must be a UUID`);
+  }
+  return result.data;
+}
+
 export class AgentManager {
   private readonly clients = new Map<AgentProvider, AgentClient>();
   private readonly agents = new Map<string, ActiveManagedAgent>();
@@ -238,9 +249,13 @@ export class AgentManager {
   }
 
   subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void {
+    const targetAgentId =
+      options?.agentId == null
+        ? null
+        : validateAgentId(options.agentId, "subscribe");
     const record: SubscriptionRecord = {
       callback,
-      agentId: options?.agentId ?? null,
+      agentId: targetAgentId,
     };
     this.subscribers.add(record);
 
@@ -334,7 +349,10 @@ export class AgentManager {
     options?: { labels?: Record<string, string> }
   ): Promise<ManagedAgent> {
     // Generate agent ID early so we can use it in MCP config
-    const resolvedAgentId = agentId ?? this.idFactory();
+    const resolvedAgentId = validateAgentId(
+      agentId ?? this.idFactory(),
+      "createAgent"
+    );
     const normalizedConfig = await this.normalizeConfig(config, {
       labels: options?.labels,
       agentId: resolvedAgentId,
@@ -364,6 +382,10 @@ export class AgentManager {
       labels?: Record<string, string>;
     }
   ): Promise<ManagedAgent> {
+    const resolvedAgentId = validateAgentId(
+      agentId ?? this.idFactory(),
+      "resumeAgent"
+    );
     const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
     const mergedConfig = {
       ...metadata,
@@ -380,7 +402,7 @@ export class AgentManager {
     return this.registerSession(
       session,
       normalizedConfig,
-      agentId ?? this.idFactory(),
+      resolvedAgentId,
       options
     );
   }
@@ -981,13 +1003,14 @@ export class AgentManager {
       labels?: Record<string, string>;
     }
   ): Promise<ManagedAgent> {
-    if (this.agents.has(agentId)) {
-      throw new Error(`Agent with id ${agentId} already exists`);
+    const resolvedAgentId = validateAgentId(agentId, "registerSession");
+    if (this.agents.has(resolvedAgentId)) {
+      throw new Error(`Agent with id ${resolvedAgentId} already exists`);
     }
 
     const now = new Date();
     const managed = {
-      id: agentId,
+      id: resolvedAgentId,
       provider: config.provider,
       cwd: config.cwd,
       session,
@@ -1010,9 +1033,9 @@ export class AgentManager {
       labels: options?.labels ?? {},
     } as ActiveManagedAgent;
 
-    this.agents.set(agentId, managed);
+    this.agents.set(resolvedAgentId, managed);
     // Initialize previousStatus to track transitions
-    this.previousStatuses.set(agentId, managed.lifecycle);
+    this.previousStatuses.set(resolvedAgentId, managed.lifecycle);
     await this.refreshRuntimeInfo(managed);
     await this.persistSnapshot(managed, {
       title: config.title ?? null,
@@ -1319,6 +1342,20 @@ export class AgentManager {
     // Always resolve cwd to absolute path for consistent history file lookup
     if (normalized.cwd) {
       normalized.cwd = resolve(normalized.cwd);
+      try {
+        const cwdStats = await stat(normalized.cwd);
+        if (!cwdStats.isDirectory()) {
+          throw new Error(`Working directory is not a directory: ${normalized.cwd}`);
+        }
+      } catch (error) {
+        if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT") {
+          throw new Error(`Working directory does not exist: ${normalized.cwd}`);
+        }
+        if (error instanceof Error) {
+          throw error;
+        }
+        throw new Error(`Failed to access working directory: ${normalized.cwd}`);
+      }
     }
 
     if (typeof normalized.model === "string") {
@@ -1338,9 +1375,10 @@ export class AgentManager {
   }
 
   private requireAgent(id: string): ActiveManagedAgent {
-    const agent = this.agents.get(id);
+    const normalizedId = validateAgentId(id, "requireAgent");
+    const agent = this.agents.get(normalizedId);
     if (!agent) {
-      throw new Error(`Unknown agent '${id}'`);
+      throw new Error(`Unknown agent '${normalizedId}'`);
     }
     return agent;
   }

packages/server/src/server/agent/agent-mcp.e2e.test.ts

@@ -90,7 +90,6 @@ describe("agent MCP end-to-end (offline)", () => {
         mcpDebug: false,
         agentClients: createTestAgentClients(),
         agentStoragePath: path.join(paseoHome, "agents"),
-        openrouterApiKey: null,
       };
 
       const daemon = await createPaseoDaemon(daemonConfig, pino({ level: "silent" }));

packages/server/src/server/agent/mcp-server.test.ts

@@ -1,4 +1,7 @@
 import { describe, expect, it, vi } from "vitest";
+import { mkdtemp, mkdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
 
 import { createTestLogger } from "../../test-utils/test-logger.js";
 import { createAgentMcpServer } from "./mcp-server.js";
@@ -48,6 +51,7 @@ function createTestDeps(): TestDeps {
 
 describe("create_agent MCP tool", () => {
   const logger = createTestLogger();
+  const existingCwd = process.cwd();
 
   it("requires a concise title no longer than 60 characters", async () => {
     const { agentManager, agentStorage } = createTestDeps();
@@ -56,26 +60,59 @@ describe("create_agent MCP tool", () => {
     expect(tool).toBeDefined();
 
     const missingTitle = await tool.inputSchema.safeParseAsync({
-      cwd: "/tmp/repo",
+      cwd: existingCwd,
       initialMode: "default",
+      initialPrompt: "test",
     });
     expect(missingTitle.success).toBe(false);
     expect(missingTitle.error.issues[0].path).toEqual(["title"]);
 
     const tooLong = await tool.inputSchema.safeParseAsync({
-      cwd: "/tmp/repo",
+      cwd: existingCwd,
       initialMode: "default",
       title: "x".repeat(61),
+      initialPrompt: "test",
     });
     expect(tooLong.success).toBe(false);
     expect(tooLong.error.issues[0].path).toEqual(["title"]);
 
     const ok = await tool.inputSchema.safeParseAsync({
-      cwd: "/tmp/repo",
+      cwd: existingCwd,
+      initialMode: "default",
+      title: "Short title",
+      initialPrompt: "test",
+    });
+    expect(ok.success).toBe(true);
+  });
+
+  it("requires initialPrompt", async () => {
+    const { agentManager, agentStorage } = createTestDeps();
+    const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
+    const tool = (server as any)._registeredTools["create_agent"];
+    const parsed = await tool.inputSchema.safeParseAsync({
+      cwd: existingCwd,
       initialMode: "default",
       title: "Short title",
     });
-    expect(ok.success).toBe(true);
+    expect(parsed.success).toBe(false);
+    expect(parsed.error.issues.some((issue: { path: string[] }) => issue.path[0] === "initialPrompt")).toBe(true);
+  });
+
+  it("surfaces createAgent validation failures", async () => {
+    const { agentManager, agentStorage, spies } = createTestDeps();
+    spies.agentManager.createAgent.mockRejectedValue(
+      new Error("Working directory does not exist: /path/that/does/not/exist")
+    );
+    const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
+    const tool = (server as any)._registeredTools["create_agent"];
+
+    await expect(
+      tool.callback({
+        cwd: "/path/that/does/not/exist",
+        title: "Short title",
+        initialPrompt: "Do work",
+      })
+    ).rejects.toThrow("Working directory does not exist");
   });
 
   it("passes caller-provided titles directly into createAgent", async () => {
@@ -86,20 +123,24 @@ describe("create_agent MCP tool", () => {
       lifecycle: "idle",
       currentModeId: null,
       availableModes: [],
+      config: { title: "Fix auth bug" },
     } as ManagedAgent);
 
     const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
     const tool = (server as any)._registeredTools["create_agent"];
     await tool.callback({
-      cwd: "/tmp/repo",
+      cwd: existingCwd,
       title: "  Fix auth bug  ",
+      initialPrompt: "Do work",
     });
 
     expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
       expect.objectContaining({
-        cwd: "/tmp/repo",
+        cwd: existingCwd,
         title: "Fix auth bug",
-      })
+      }),
+      undefined,
+      undefined
     );
   });
 
@@ -111,19 +152,127 @@ describe("create_agent MCP tool", () => {
       lifecycle: "idle",
       currentModeId: null,
       availableModes: [],
+      config: { title: "Fix auth" },
     } as ManagedAgent);
 
     const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
     const tool = (server as any)._registeredTools["create_agent"];
     await tool.callback({
-      cwd: "/tmp/repo",
+      cwd: existingCwd,
       title: "  Fix auth  ",
+      initialPrompt: "Do work",
     });
 
     expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
       expect.objectContaining({
         title: "Fix auth",
+      }),
+      undefined,
+      undefined
+    );
+  });
+
+  it("allows caller agents to override cwd and applies caller context labels", async () => {
+    const { agentManager, agentStorage, spies } = createTestDeps();
+    const baseDir = await mkdtemp(join(tmpdir(), "paseo-mcp-test-"));
+    const subdir = join(baseDir, "subdir");
+    await mkdir(subdir, { recursive: true });
+    spies.agentManager.getAgent.mockReturnValue({
+      id: "voice-agent",
+      cwd: baseDir,
+      provider: "codex",
+      currentModeId: "full-access",
+    } as ManagedAgent);
+    spies.agentManager.createAgent.mockResolvedValue({
+      id: "child-agent",
+      cwd: subdir,
+      lifecycle: "idle",
+      currentModeId: null,
+      availableModes: [],
+      config: { title: "Child" },
+    } as ManagedAgent);
+
+    const server = await createAgentMcpServer({
+      agentManager,
+      agentStorage,
+      callerAgentId: "voice-agent",
+      resolveCallerContext: () => ({
+        childAgentDefaultLabels: { ui: "true" },
+        allowCustomCwd: true,
+      }),
+      logger,
+    });
+
+    const tool = (server as any)._registeredTools["create_agent"];
+    await tool.callback({
+      cwd: "subdir",
+      title: "Child",
+      agentType: "codex",
+      initialPrompt: "Do work",
+    });
+
+    expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
+      expect.objectContaining({
+        cwd: subdir,
+      }),
+      undefined,
+      { labels: { ui: "true" } }
+    );
+    await rm(baseDir, { recursive: true, force: true });
+  });
+});
+
+describe("speak MCP tool", () => {
+  const logger = createTestLogger();
+
+  it("invokes registered speak handler for caller agent", async () => {
+    const { agentManager, agentStorage } = createTestDeps();
+    const speak = vi.fn().mockResolvedValue(undefined);
+    const server = await createAgentMcpServer({
+      agentManager,
+      agentStorage,
+      callerAgentId: "voice-agent-1",
+      enableVoiceTools: true,
+      resolveSpeakHandler: () => speak,
+      logger,
+    });
+    const tool = (server as any)._registeredTools["speak"];
+    expect(tool).toBeDefined();
+
+    await tool.callback({ text: "Hello from voice agent." });
+    expect(speak).toHaveBeenCalledWith(
+      expect.objectContaining({
+        text: "Hello from voice agent.",
+        callerAgentId: "voice-agent-1",
       })
     );
   });
+
+  it("fails when no speak handler exists", async () => {
+    const { agentManager, agentStorage } = createTestDeps();
+    const server = await createAgentMcpServer({
+      agentManager,
+      agentStorage,
+      callerAgentId: "voice-agent-2",
+      enableVoiceTools: true,
+      resolveSpeakHandler: () => null,
+      logger,
+    });
+    const tool = (server as any)._registeredTools["speak"];
+    await expect(tool.callback({ text: "Hello." })).rejects.toThrow(
+      "No speak handler registered for caller agent"
+    );
+  });
+
+  it("does not register speak tool unless voice tools are enabled", async () => {
+    const { agentManager, agentStorage } = createTestDeps();
+    const server = await createAgentMcpServer({
+      agentManager,
+      agentStorage,
+      callerAgentId: "agent-no-voice",
+      logger,
+    });
+    const tool = (server as any)._registeredTools["speak"];
+    expect(tool).toBeUndefined();
+  });
 });

packages/server/src/server/agent/mcp-server.ts

@@ -1,9 +1,12 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
-import { homedir } from "node:os";
-import { resolve } from "node:path";
 import { ensureValidJson } from "../json-utils.js";
 import type { Logger } from "pino";
+import type { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js";
+import type {
+  ServerNotification,
+  ServerRequest,
+} from "@modelcontextprotocol/sdk/types.js";
 
 import type {
   AgentPromptInput,
@@ -28,6 +31,11 @@ import { AgentStorage } from "./agent-storage.js";
 import { createWorktree } from "../../utils/worktree.js";
 import { WaitForAgentTracker } from "./wait-for-agent-tracker.js";
 import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js";
+import type {
+  VoiceCallerContext,
+  VoiceSpeakHandler,
+} from "../voice-types.js";
+import { expandUserPath, resolvePathFromBase } from "../path-utils.js";
 
 export interface AgentMcpServerOptions {
   agentManager: AgentManager;
@@ -38,6 +46,17 @@ export interface AgentMcpServerOptions {
    * Used for cwd/mode inheritance when agents spawn child agents.
    */
   callerAgentId?: string;
+  /**
+   * Optional resolver for session-bound speak handlers.
+   * Used by hidden voice agents to narrate through daemon-managed TTS.
+   */
+  resolveSpeakHandler?: (
+    callerAgentId: string
+  ) => VoiceSpeakHandler | null;
+  resolveCallerContext?: (
+    callerAgentId: string
+  ) => VoiceCallerContext | null;
+  enableVoiceTools?: boolean;
   logger: Logger;
 }
 
@@ -100,11 +119,25 @@ const AgentStatusEnum = z.enum([
 // 50 seconds - surface friendly message before SDK tool timeout (~60s)
 const AGENT_WAIT_TIMEOUT_MS = 50000;
 
-function expandPath(path: string): string {
-  if (path.startsWith("~/") || path === "~") {
-    return resolve(homedir(), path.slice(2));
+type McpToolContext = RequestHandlerExtra<ServerRequest, ServerNotification>;
+
+function resolveChildAgentCwd(params: {
+  parentCwd: string;
+  requestedCwd?: string;
+  lockedCwd?: string;
+  allowCustomCwd: boolean;
+}): string {
+  const lockedCwd = params.lockedCwd?.trim();
+  if (lockedCwd) {
+    return expandUserPath(lockedCwd);
   }
-  return resolve(path);
+
+  const requestedCwd = params.requestedCwd?.trim();
+  if (!requestedCwd || !params.allowCustomCwd) {
+    return params.parentCwd;
+  }
+
+  return resolvePathFromBase(params.parentCwd, requestedCwd);
 }
 
 /**
@@ -252,9 +285,17 @@ async function serializeSnapshotWithMetadata(
 export async function createAgentMcpServer(
   options: AgentMcpServerOptions
 ): Promise<McpServer> {
-  const { agentManager, agentStorage, callerAgentId, logger } = options;
+  const {
+    agentManager,
+    agentStorage,
+    callerAgentId,
+    resolveSpeakHandler,
+    resolveCallerContext,
+    logger,
+  } = options;
   const childLogger = logger.child({ module: "agent", component: "mcp-server" });
   const waitTracker = new WaitForAgentTracker(logger);
+  const callerContext = callerAgentId ? resolveCallerContext?.(callerAgentId) ?? null : null;
 
   const server = new McpServer({
     name: "agent-mcp",
@@ -262,6 +303,12 @@ export async function createAgentMcpServer(
   });
 
   const agentToAgentInputSchema = {
+    cwd: z
+      .string()
+      .optional()
+      .describe(
+        "Optional working directory. Defaults to the caller agent working directory."
+      ),
     title: z
       .string()
       .trim()
@@ -275,9 +322,10 @@ export async function createAgentMcpServer(
     ),
     initialPrompt: z
       .string()
-      .optional()
+      .trim()
+      .min(1, "initialPrompt is required")
       .describe(
-        "Optional task to start immediately after creation (non-blocking)."
+        "Required first task to run immediately after creation."
       ),
     background: z
       .boolean()
@@ -307,9 +355,10 @@ export async function createAgentMcpServer(
     ),
     initialPrompt: z
       .string()
-      .optional()
+      .trim()
+      .min(1, "initialPrompt is required")
       .describe(
-        "Optional task to start immediately after creation (non-blocking)."
+        "Required first task to run immediately after creation."
       ),
     initialMode: z
       .string()
@@ -338,6 +387,50 @@ export async function createAgentMcpServer(
   const createAgentInputSchema = callerAgentId
     ? agentToAgentInputSchema
     : topLevelInputSchema;
+  const agentToAgentCreateAgentArgsSchema = z.object(agentToAgentInputSchema);
+  const topLevelCreateAgentArgsSchema = z.object({
+    ...topLevelInputSchema,
+    initialMode: topLevelInputSchema.initialMode.optional(),
+  });
+
+  if (options.enableVoiceTools || callerContext?.enableVoiceTools) {
+    server.registerTool(
+      "speak",
+      {
+        title: "Speak",
+        description:
+          "Speak text to the user via daemon-managed voice output. Blocks until playback completes.",
+        inputSchema: {
+          text: z
+            .string()
+            .trim()
+            .min(1, "text is required")
+            .max(4000, "text must be 4000 characters or fewer"),
+        },
+        outputSchema: {
+          ok: z.boolean(),
+        },
+      },
+      async (args, context?: McpToolContext) => {
+        if (!callerAgentId) {
+          throw new Error("speak is only available to agent-scoped MCP sessions");
+        }
+        const handler = resolveSpeakHandler?.(callerAgentId) ?? null;
+        if (!handler) {
+          throw new Error(`No speak handler registered for caller agent '${callerAgentId}'`);
+        }
+        await handler({
+          text: args.text,
+          callerAgentId,
+          signal: context?.signal,
+        });
+        return {
+          content: [],
+          structuredContent: ensureValidJson({ ok: true }),
+        };
+      }
+    );
+  }
 
   server.registerTool(
     "create_agent",
@@ -363,33 +456,32 @@ export async function createAgentMcpServer(
         permission: AgentPermissionRequestPayloadSchema.nullable().optional(),
       },
     },
-    async (args) => {
-      const {
-        agentType,
-        initialPrompt,
-        background = false,
-        title,
-      } = args as {
-        cwd?: string;
-        agentType?: AgentProvider;
-        initialPrompt?: string;
-        initialMode?: string;
-        worktreeName?: string;
-        background?: boolean;
-        title: string;
-      };
+    async (args: unknown) => {
+      let provider: AgentProvider;
+      let initialPrompt: string;
+      let background = false;
+      let normalizedTitle: string | null;
 
       let resolvedCwd: string;
       let resolvedMode: string | undefined;
 
       if (callerAgentId) {
+        const callerArgs = agentToAgentCreateAgentArgsSchema.parse(args);
+        provider = callerArgs.agentType ?? "claude";
+        initialPrompt = callerArgs.initialPrompt;
+        background = callerArgs.background ?? false;
+        normalizedTitle = callerArgs.title.trim();
+
         const parentAgent = agentManager.getAgent(callerAgentId);
         if (!parentAgent) {
           throw new Error(`Parent agent ${callerAgentId} not found`);
         }
-        resolvedCwd = parentAgent.cwd;
-
-        const provider: AgentProvider = agentType ?? "claude";
+        resolvedCwd = resolveChildAgentCwd({
+          parentCwd: parentAgent.cwd,
+          requestedCwd: callerArgs.cwd,
+          lockedCwd: callerContext?.lockedCwd,
+          allowCustomCwd: callerContext?.allowCustomCwd ?? true,
+        });
         const parentMode = parentAgent.currentModeId;
         if (parentMode) {
           resolvedMode = mapModeAcrossProviders(
@@ -399,12 +491,11 @@ export async function createAgentMcpServer(
           );
         }
       } else {
-        const topLevelArgs = args as unknown as {
-          cwd: string;
-          initialMode: string;
-          worktreeName?: string;
-          baseBranch?: string;
-        };
+        const topLevelArgs = topLevelCreateAgentArgsSchema.parse(args);
+        provider = topLevelArgs.agentType ?? "claude";
+        initialPrompt = topLevelArgs.initialPrompt;
+        background = topLevelArgs.background ?? false;
+        normalizedTitle = topLevelArgs.title.trim();
         const {
           cwd,
           initialMode,
@@ -412,7 +503,7 @@ export async function createAgentMcpServer(
           baseBranch,
         } = topLevelArgs;
 
-        resolvedCwd = expandPath(cwd);
+        resolvedCwd = expandUserPath(cwd);
 
         if (worktreeName) {
           if (!baseBranch) {
@@ -431,75 +522,78 @@ export async function createAgentMcpServer(
         resolvedMode = initialMode;
       }
 
-      const provider: AgentProvider = agentType ?? "claude";
-      const normalizedTitle = title?.trim() ?? null;
-      const snapshot = await agentManager.createAgent({
-        provider,
-        cwd: resolvedCwd,
-        modeId: resolvedMode,
-        title: normalizedTitle ?? undefined,
+      const childAgentDefaultLabels =
+        callerAgentId && callerContext?.childAgentDefaultLabels
+          ? callerContext.childAgentDefaultLabels
+          : undefined;
+      const snapshot = await agentManager.createAgent(
+        {
+          provider,
+          cwd: resolvedCwd,
+          modeId: resolvedMode,
+          title: normalizedTitle ?? undefined,
+        },
+        undefined,
+        childAgentDefaultLabels ? { labels: childAgentDefaultLabels } : undefined
+      );
+
+      const trimmedPrompt = initialPrompt.trim();
+      scheduleAgentMetadataGeneration({
+        agentManager,
+        agentId: snapshot.id,
+        cwd: snapshot.cwd,
+        initialPrompt: trimmedPrompt,
+        explicitTitle: snapshot.config.title,
+        paseoHome: options.paseoHome,
+        logger: childLogger,
       });
 
-      const trimmedPrompt = initialPrompt?.trim();
-      if (trimmedPrompt) {
-        scheduleAgentMetadataGeneration({
-          agentManager,
-          agentId: snapshot.id,
-          cwd: snapshot.cwd,
-          initialPrompt: trimmedPrompt,
-          explicitTitle: snapshot.config.title,
-          paseoHome: options.paseoHome,
-          logger: childLogger,
-        });
-
-        try {
-          agentManager.recordUserMessage(snapshot.id, trimmedPrompt);
-        } catch (error) {
-          childLogger.error(
-            { err: error, agentId: snapshot.id },
-            "Failed to record initial prompt"
-          );
-        }
-
-        try {
-          startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger);
-
-          // If not running in background, wait for completion
-          if (!background) {
-            const result = await waitForAgentWithTimeout(
-              agentManager,
-              snapshot.id,
-              { waitForActive: true }
-            );
-
-            const responseData = {
-              agentId: snapshot.id,
-              type: provider,
-              status: result.status,
-              cwd: snapshot.cwd,
-              currentModeId: snapshot.currentModeId,
-              availableModes: snapshot.availableModes,
-              lastMessage: result.lastMessage,
-              permission: sanitizePermissionRequest(result.permission),
-            };
-            const validJson = ensureValidJson(responseData);
-
-            const response = {
-              content: [],
-              structuredContent: validJson,
-            };
-            return response;
-          }
-        } catch (error) {
-          childLogger.error(
-            { err: error, agentId: snapshot.id },
-            "Failed to run initial prompt"
-          );
-        }
-      } else {
+      try {
+        agentManager.recordUserMessage(snapshot.id, trimmedPrompt);
+      } catch (error) {
+        childLogger.error(
+          { err: error, agentId: snapshot.id },
+          "Failed to record initial prompt"
+        );
       }
 
-      // Return immediately if background=true or no initialPrompt
+      try {
+        startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger);
+
+        // If not running in background, wait for completion
+        if (!background) {
+          const result = await waitForAgentWithTimeout(
+            agentManager,
+            snapshot.id,
+            { waitForActive: true }
+          );
+
+          const responseData = {
+            agentId: snapshot.id,
+            type: provider,
+            status: result.status,
+            cwd: snapshot.cwd,
+            currentModeId: snapshot.currentModeId,
+            availableModes: snapshot.availableModes,
+            lastMessage: result.lastMessage,
+            permission: sanitizePermissionRequest(result.permission),
+          };
+          const validJson = ensureValidJson(responseData);
+
+          const response = {
+            content: [],
+            structuredContent: validJson,
+          };
+          return response;
+        }
+      } catch (error) {
+        childLogger.error(
+          { err: error, agentId: snapshot.id },
+          "Failed to run initial prompt"
+        );
+      }
+
+      // Return immediately if background=true
       const response = {
         content: [],
         structuredContent: ensureValidJson({

packages/server/src/server/agent/provider-manifest.ts

@@ -7,6 +7,11 @@ export interface AgentProviderDefinition {
   description: string;
   defaultModeId: string | null;
   modes: AgentMode[];
+  voice?: {
+    enabled: boolean;
+    defaultModeId: string;
+    defaultModel?: string;
+  };
 }
 
 const CLAUDE_MODES: AgentMode[] = [
@@ -68,6 +73,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [
       "Anthropic's multi-tool assistant with MCP support, streaming, and deep reasoning",
     defaultModeId: "default",
     modes: CLAUDE_MODES,
+    voice: {
+      enabled: true,
+      defaultModeId: "default",
+      defaultModel: "haiku",
+    },
   },
   {
     id: "codex",
@@ -76,6 +86,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [
       "OpenAI's Codex workspace agent with sandbox controls and optional network access",
     defaultModeId: "auto",
     modes: CODEX_MODES,
+    voice: {
+      enabled: true,
+      defaultModeId: "read-only",
+      defaultModel: "gpt-5.2-mini",
+    },
   },
   {
     id: "opencode",
@@ -84,6 +99,10 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [
       "Open-source coding assistant with multi-provider model support",
     defaultModeId: "default",
     modes: OPENCODE_MODES,
+    voice: {
+      enabled: true,
+      defaultModeId: "default",
+    },
   },
 ];
 

packages/server/src/server/bootstrap.smoke.test.ts

@@ -10,7 +10,16 @@ import { createTestAgentClients } from "./test-utils/fake-agent-client.js";
 
 describe("paseo daemon bootstrap", () => {
   test("starts and serves health endpoint", async () => {
-    const daemonHandle = await createTestPaseoDaemon();
+    const daemonHandle = await createTestPaseoDaemon({
+      openai: { apiKey: "test-openai-api-key" },
+      speech: {
+        providers: {
+          dictationStt: { provider: "openai", explicit: true },
+          voiceStt: { provider: "openai", explicit: true },
+          voiceTts: { provider: "openai", explicit: true },
+        },
+      },
+    });
     try {
       const response = await fetch(
         `http://127.0.0.1:${daemonHandle.port}/api/health`,
@@ -49,11 +58,12 @@ describe("paseo daemon bootstrap", () => {
       appBaseUrl: "https://app.paseo.sh",
       openai: undefined,
       speech: {
-        dictationSttProvider: "openai",
-        voiceSttProvider: "openai",
-        voiceTtsProvider: "openai",
+        providers: {
+          dictationStt: { provider: "openai", explicit: true },
+          voiceStt: { provider: "openai", explicit: true },
+          voiceTts: { provider: "openai", explicit: true },
+        },
       },
-      openrouterApiKey: null,
     };
 
     try {

packages/server/src/server/bootstrap.ts

@@ -3,6 +3,7 @@ import { createServer as createHTTPServer } from "http";
 import { createReadStream, unlinkSync, existsSync } from "fs";
 import { stat } from "fs/promises";
 import { randomUUID } from "node:crypto";
+import path from "node:path";
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
 import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
 import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
@@ -40,20 +41,10 @@ function parseListenString(listen: string): ListenTarget {
 
 import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
 import { DownloadTokenStore } from "./file-download/token-store.js";
-import { OpenAISTT, type STTConfig } from "./speech/providers/openai/stt.js";
-import { OpenAITTS, type TTSConfig } from "./speech/providers/openai/tts.js";
-import { OpenAIRealtimeTranscriptionSession } from "./speech/providers/openai/realtime-transcription-session.js";
-import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
-import { SherpaOnlineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-online-recognizer.js";
-import { SherpaOfflineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-offline-recognizer.js";
-import { SherpaOnnxSTT } from "./speech/providers/local/sherpa/sherpa-stt.js";
-import { SherpaOnnxParakeetSTT } from "./speech/providers/local/sherpa/sherpa-parakeet-stt.js";
-import { SherpaOnnxTTS } from "./speech/providers/local/sherpa/sherpa-tts.js";
-import { SherpaRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-realtime-session.js";
-import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-parakeet-realtime-session.js";
-import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/providers/local/sherpa/model-downloader.js";
-import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js";
-import { PocketTtsOnnxTTS } from "./speech/providers/local/pocket/pocket-tts-onnx.js";
+import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js";
+import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js";
+import type { RequestedSpeechProviders } from "./speech/speech-types.js";
+import { initializeSpeechRuntime } from "./speech/speech-runtime.js";
 import { AgentManager } from "./agent/agent-manager.js";
 import { AgentStorage } from "./agent/agent-storage.js";
 import { attachAgentStoragePersistence } from "./persistence-hooks.js";
@@ -73,35 +64,50 @@ import type {
   AgentClient,
   AgentProvider,
 } from "./agent/agent-sdk-types.js";
+import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js";
 import { acquirePidLock, releasePidLock } from "./pid-lock.js";
 import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js";
+import { createVoiceMcpBridgeSocketServer, type VoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js";
 
 type AgentMcpTransportMap = Map<string, StreamableHTTPServerTransport>;
 
-export type PaseoOpenAIConfig = {
-  apiKey?: string;
-  stt?: Partial<STTConfig> & { apiKey?: string };
-  tts?: Partial<TTSConfig> & { apiKey?: string };
-};
+function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } {
+  const explicit = process.env.PASEO_BIN_PATH?.trim();
+  if (explicit) {
+    const resolved = { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] };
+    logger.info({ source: "PASEO_BIN_PATH", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
+    return resolved;
+  }
 
-export type PaseoSherpaOnnxConfig = {
-  modelsDir: string;
-  autoDownload?: boolean;
-  stt?: {
-    preset?: string;
-  };
-  tts?: {
-    preset?: string;
-    speakerId?: number;
-    speed?: number;
-  };
-};
+  const argv1 = process.argv[1]?.trim();
+  if (!argv1) {
+    logger.warn("Could not resolve argv[1] for voice MCP bridge; falling back to 'paseo'");
+    const resolved = { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] };
+    logger.info({ source: "fallback", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
+    return resolved;
+  }
+
+  const base = path.basename(argv1).toLowerCase();
+  if (base.includes("tsx") && process.argv[2]) {
+    const resolved = {
+      command: process.execPath,
+      baseArgs: [argv1, process.argv[2], "__paseo_voice_mcp_bridge"],
+    };
+    logger.info({ source: "tsx", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
+    return resolved;
+  }
+
+  const resolved = { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] };
+  logger.info({ source: "argv", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
+  return resolved;
+}
+
+export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig;
+export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig;
 
 export type PaseoSpeechConfig = {
-  dictationSttProvider?: "openai" | "local";
-  voiceSttProvider?: "openai" | "local";
-  voiceTtsProvider?: "openai" | "local";
-  sherpaOnnx?: PaseoSherpaOnnxConfig;
+  providers: RequestedSpeechProviders;
+  local?: PaseoLocalSpeechConfig;
 };
 
 export type PaseoDaemonConfig = {
@@ -120,7 +126,8 @@ export type PaseoDaemonConfig = {
   appBaseUrl?: string;
   openai?: PaseoOpenAIConfig;
   speech?: PaseoSpeechConfig;
-  openrouterApiKey?: string | null;
+  voiceLlmProvider?: AgentProvider | null;
+  voiceLlmProviderExplicit?: boolean;
   voiceLlmModel?: string | null;
   dictationFinalTimeoutMs?: number;
   downloadTokenTtlMs?: number;
@@ -282,12 +289,110 @@ export async function createPaseoDaemon(
     `Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`
   );
 
+  const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null;
+  const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false;
+  const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS
+    .filter((definition) => definition.voice?.enabled)
+    .map((definition) => definition.id as AgentProvider);
+  logger.info(
+    {
+      requestedVoiceLlmProvider,
+      voiceLlmProviderExplicit,
+      voiceEnabledProviders,
+    },
+    "Voice LLM provider reconciliation started"
+  );
+
+  const providerClients = createAllClients(logger);
+  Object.assign(providerClients, config.agentClients);
+  const voiceLlmAvailability = Object.fromEntries(
+    voiceEnabledProviders.map((provider) => [provider, false])
+  ) as Record<AgentProvider, boolean>;
+  for (const provider of voiceEnabledProviders) {
+    try {
+      voiceLlmAvailability[provider] = await providerClients[provider].isAvailable();
+    } catch (error) {
+      logger.warn({ err: error, provider }, "Voice LLM provider availability check failed");
+      voiceLlmAvailability[provider] = false;
+    }
+  }
+
+  let resolvedVoiceLlmProvider: AgentProvider | null = null;
+  if (requestedVoiceLlmProvider) {
+    if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) {
+      logger.error(
+        { provider: requestedVoiceLlmProvider, voiceEnabledProviders },
+        "Configured voice LLM provider does not support voice mode"
+      );
+      throw new Error(
+        `Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode`
+      );
+    }
+    if (!voiceLlmAvailability[requestedVoiceLlmProvider]) {
+      logger.error(
+        { provider: requestedVoiceLlmProvider, voiceLlmAvailability },
+        "Configured voice LLM provider is unavailable"
+      );
+      throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`);
+    }
+    resolvedVoiceLlmProvider = requestedVoiceLlmProvider;
+  } else {
+    resolvedVoiceLlmProvider =
+      voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null;
+  }
+
+  let resolvedVoiceLlmModeId: string | null = null;
+  let resolvedVoiceLlmModel: string | null = null;
+
+  if (!resolvedVoiceLlmProvider) {
+    if (voiceLlmProviderExplicit) {
+      logger.error(
+        { requestedVoiceLlmProvider, voiceLlmAvailability },
+        "No voice LLM provider available"
+      );
+      throw new Error("No voice LLM provider available");
+    }
+    logger.warn(
+      { requestedVoiceLlmProvider, voiceLlmAvailability },
+      "No default voice LLM provider available; voice mode will be disabled until a provider is configured"
+    );
+  } else {
+    const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
+      (definition) => definition.id === resolvedVoiceLlmProvider
+    );
+    if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
+      throw new Error(
+        `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
+      );
+    }
+    resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
+    resolvedVoiceLlmModel =
+      config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;
+  }
+
+  logger.info(
+    {
+      requestedVoiceLlmProvider,
+      voiceLlmProviderExplicit,
+      resolvedVoiceLlmProvider,
+      resolvedVoiceLlmModeId,
+      resolvedVoiceLlmModel,
+      voiceLlmAvailability,
+    },
+    "Voice LLM provider reconciliation completed"
+  );
+  let wsServer: VoiceAssistantWebSocketServer | null = null;
+  let voiceMcpBridgeServer: VoiceMcpBridgeSocketServer | null = null;
+
   // Create in-memory transport for Session's Agent MCP client (voice assistant tools)
   const createInMemoryAgentMcpTransport = async (): Promise<InMemoryTransport> => {
     const agentMcpServer = await createAgentMcpServer({
       agentManager,
       agentStorage,
       paseoHome: config.paseoHome,
+      enableVoiceTools: false,
+      resolveSpeakHandler: (callerAgentId) => wsServer?.resolveVoiceSpeakHandler(callerAgentId) ?? null,
+      resolveCallerContext: (callerAgentId) => wsServer?.resolveVoiceCallerContext(callerAgentId) ?? null,
       logger,
     });
 
@@ -309,6 +414,9 @@ export async function createPaseoDaemon(
         agentStorage,
         paseoHome: config.paseoHome,
         callerAgentId,
+        enableVoiceTools: false,
+        resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
+        resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
         logger,
       });
 
@@ -414,385 +522,37 @@ export async function createPaseoDaemon(
     logger.info("Agent MCP HTTP endpoint disabled");
   }
 
-
-  let sttService: SpeechToTextProvider | null = null;
-  let ttsService: TextToSpeechProvider | null = null;
-  let dictationSttService: SpeechToTextProvider | null = null;
-
-  let sherpaOnline: SherpaOnlineRecognizerEngine | null = null;
-  let sherpaOffline: SherpaOfflineRecognizerEngine | null = null;
-  let sherpaTts: TextToSpeechProvider | null = null;
-
-  const openaiApiKey = config.openai?.apiKey;
-  const speechConfig = config.speech ?? null;
-  const sherpaConfig = speechConfig?.sherpaOnnx ?? null;
-
-  const voiceSttProvider = speechConfig?.voiceSttProvider ?? "local";
-  const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "local";
-  const dictationSttProvider = speechConfig?.dictationSttProvider ?? "local";
-
-  const wantsLocalDictation = dictationSttProvider === "local";
-  const wantsLocalVoiceStt = voiceSttProvider === "local";
-  const wantsLocalVoiceTts = voiceTtsProvider === "local";
-
-  const openaiSttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey;
-  const openaiTtsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey;
-  const openaiDictationApiKey = openaiApiKey;
-
-  const missingOpenAiCredentialsFor: string[] = [];
-  if (voiceSttProvider === "openai" && !openaiSttApiKey) {
-    missingOpenAiCredentialsFor.push("voice.stt");
-  }
-  if (voiceTtsProvider === "openai" && !openaiTtsApiKey) {
-    missingOpenAiCredentialsFor.push("voice.tts");
-  }
-  if (dictationSttProvider === "openai" && !openaiDictationApiKey) {
-    missingOpenAiCredentialsFor.push("dictation.stt");
-  }
-
-  if (missingOpenAiCredentialsFor.length > 0) {
-    logger.error(
-      {
-        requestedProviders: {
-          dictationStt: dictationSttProvider,
-          voiceStt: voiceSttProvider,
-          voiceTts: voiceTtsProvider,
-        },
-        missingOpenAiCredentialsFor,
-      },
-      "Invalid speech configuration: OpenAI provider selected but credentials are missing"
-    );
-    throw new Error(
-      `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
-    );
-  }
-
-  logger.info(
-    {
-      requestedProviders: {
-        dictationStt: dictationSttProvider,
-        voiceStt: voiceSttProvider,
-        voiceTts: voiceTtsProvider,
-      },
-      availability: {
-        openai: {
-          stt: Boolean(openaiSttApiKey),
-          tts: Boolean(openaiTtsApiKey),
-          dictationStt: Boolean(openaiDictationApiKey),
-        },
-        local: {
-          configured: Boolean(sherpaConfig),
-          modelsDir: sherpaConfig?.modelsDir ?? null,
-          autoDownload: sherpaConfig?.autoDownload ?? null,
-        },
-      },
-    },
-    "Speech provider reconciliation started"
-  );
-
-  if ((wantsLocalDictation || wantsLocalVoiceStt || wantsLocalVoiceTts) && sherpaConfig) {
-    const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true);
-    let sttPreset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim();
-    if (
-      sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" &&
-      sttPreset !== "paraformer-bilingual-zh-en" &&
-      sttPreset !== "parakeet-tdt-0.6b-v3-int8"
-    ) {
-      throw new Error(`Unknown local STT preset: ${sttPreset}`);
-    }
-
-    let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
-    if (
-      ttsPreset !== "kitten-nano-en-v0_1-fp16" &&
-      ttsPreset !== "kokoro-en-v0_19" &&
-      ttsPreset !== "pocket-tts-onnx-int8"
-    ) {
-      throw new Error(`Unknown local TTS preset: ${ttsPreset}`);
-    }
-
-    const modelIds: SherpaOnnxModelId[] = [];
-    if (wantsLocalDictation || wantsLocalVoiceStt) {
-      modelIds.push(sttPreset as SherpaOnnxModelId);
-    }
-    if (wantsLocalVoiceTts) {
-      modelIds.push(ttsPreset as SherpaOnnxModelId);
-    }
-
-    try {
-      logger.info(
-        {
-          modelsDir: sherpaConfig.modelsDir,
-          modelIds,
-          autoDownload,
-        },
-        "Ensuring local speech models"
-      );
-      await ensureSherpaOnnxModels({
-        modelsDir: sherpaConfig.modelsDir,
-        modelIds,
-        autoDownload,
+  const voiceMcpSocketPath = path.join(config.paseoHome, "runtime", "voice-mcp.sock");
+  const voiceMcpBridgeCommand = resolveVoiceMcpBridgeCommand(logger);
+  voiceMcpBridgeServer = createVoiceMcpBridgeSocketServer({
+    socketPath: voiceMcpSocketPath,
+    logger,
+    createAgentMcpServerForCaller: async (callerAgentId) => {
+      return createAgentMcpServer({
+        agentManager,
+        agentStorage,
+        paseoHome: config.paseoHome,
+        callerAgentId,
+        enableVoiceTools: false,
+        resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
+        resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
         logger,
       });
-    } catch (err) {
-      logger.error(
-        {
-          err,
-          modelsDir: sherpaConfig.modelsDir,
-          autoDownload,
-          hint:
-            "Run: npm run dev --workspace=@getpaseo/server, then run: " +
-            "`tsx packages/server/scripts/download-speech-models.ts --models-dir <DIR> --model <MODEL_ID>`",
-        },
-        "Failed to ensure local speech models"
-      );
-    }
-  }
+    },
+  });
+  const {
+    sttService,
+    ttsService,
+    dictationSttService,
+    cleanup: cleanupSpeechRuntime,
+    localModelConfig,
+  } = await initializeSpeechRuntime({
+    logger,
+    openaiConfig: config.openai,
+    speechConfig: config.speech,
+  });
 
-  if ((wantsLocalDictation || wantsLocalVoiceStt) && sherpaConfig) {
-    let preset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim();
-    if (
-      preset !== "zipformer-bilingual-zh-en-2023-02-20" &&
-      preset !== "paraformer-bilingual-zh-en" &&
-      preset !== "parakeet-tdt-0.6b-v3-int8"
-    ) {
-      throw new Error(`Unknown local STT preset: ${preset}`);
-    }
-    const base = sherpaConfig.modelsDir;
-
-    try {
-      if (preset === "parakeet-tdt-0.6b-v3-int8") {
-        const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8");
-        sherpaOffline = new SherpaOfflineRecognizerEngine(
-          {
-            model: {
-              kind: "nemo_transducer",
-              encoder: `${modelDir}/encoder.int8.onnx`,
-              decoder: `${modelDir}/decoder.int8.onnx`,
-              joiner: `${modelDir}/joiner.int8.onnx`,
-              tokens: `${modelDir}/tokens.txt`,
-            },
-            numThreads: 2,
-            debug: 0,
-          },
-          logger
-        );
-      } else {
-        const model =
-          preset === "paraformer-bilingual-zh-en"
-            ? {
-                kind: "paraformer" as const,
-                encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`,
-                decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`,
-                tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`,
-              }
-            : {
-                kind: "transducer" as const,
-                encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`,
-                decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`,
-                joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`,
-                tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`,
-                modelType: "zipformer",
-              };
-
-        sherpaOnline = new SherpaOnlineRecognizerEngine(
-          {
-            model,
-            numThreads: 1,
-            debug: 0,
-          },
-          logger
-        );
-      }
-    } catch (err) {
-      logger.error(
-        {
-          err,
-          modelsDir: sherpaConfig.modelsDir,
-          preset,
-          hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
-        },
-        "Failed to initialize Sherpa STT (models missing or invalid)"
-      );
-      sherpaOnline = null;
-      sherpaOffline = null;
-    }
-  } else if (wantsLocalDictation || wantsLocalVoiceStt) {
-    logger.warn(
-      { configured: Boolean(sherpaConfig) },
-      "Local STT selected but local provider config is missing; STT will be unavailable"
-    );
-  }
-
-  if (wantsLocalVoiceTts && sherpaConfig) {
-    let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
-    if (
-      preset !== "kitten-nano-en-v0_1-fp16" &&
-      preset !== "kokoro-en-v0_19" &&
-      preset !== "pocket-tts-onnx-int8"
-    ) {
-      throw new Error(`Unknown local TTS preset: ${preset}`);
-    }
-    try {
-      if (preset === "pocket-tts-onnx-int8") {
-        const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8");
-        sherpaTts = await PocketTtsOnnxTTS.create(
-          {
-            modelDir,
-            precision: "int8",
-            targetChunkMs: 50,
-          },
-          logger
-        );
-      } else {
-        const modelDir = `${sherpaConfig.modelsDir}/${preset}`;
-        sherpaTts = new SherpaOnnxTTS(
-          {
-            preset: preset as any,
-            modelDir,
-            speakerId: sherpaConfig.tts?.speakerId,
-            speed: sherpaConfig.tts?.speed,
-          },
-          logger
-        );
-      }
-    } catch (err) {
-      logger.error(
-        {
-          err,
-          preset,
-          hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
-        },
-        "Failed to initialize Sherpa TTS (models missing or invalid)"
-      );
-      sherpaTts = null;
-    }
-  } else if (wantsLocalVoiceTts) {
-    logger.warn(
-      { configured: Boolean(sherpaConfig) },
-      "Local TTS selected but local provider config is missing; TTS will be unavailable"
-    );
-  }
-
-  if (wantsLocalVoiceStt && sherpaOffline) {
-    sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger);
-  } else if (wantsLocalVoiceStt && sherpaOnline) {
-    sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger);
-  }
-
-  if (wantsLocalVoiceTts && sherpaTts) {
-    ttsService = sherpaTts;
-  }
-
-  if (wantsLocalDictation && sherpaOnline) {
-    dictationSttService = {
-      id: "local",
-      createSession: () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }),
-    };
-  } else if (wantsLocalDictation && sherpaOffline) {
-    dictationSttService = {
-      id: "local",
-      createSession: () =>
-        new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }),
-    };
-  }
-
-  const needsOpenAiStt = !sttService && voiceSttProvider === "openai";
-  const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai";
-  const needsOpenAiDictation = dictationSttProvider === "openai";
-
-  if (
-    (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
-    (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey)
-  ) {
-    logger.info("OpenAI speech provider initialized");
-
-    if (needsOpenAiStt) {
-      if (openaiSttApiKey) {
-        const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {};
-        sttService = new OpenAISTT(
-          {
-            apiKey: openaiSttApiKey,
-            ...sttConfig,
-          },
-          logger
-        );
-      }
-    }
-
-    if (needsOpenAiTts) {
-      if (openaiTtsApiKey) {
-        const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {};
-        ttsService = new OpenAITTS(
-          {
-            apiKey: openaiTtsApiKey,
-            voice: "alloy",
-            model: "tts-1",
-            responseFormat: "pcm",
-            ...ttsConfig,
-          },
-          logger
-        );
-      }
-    }
-
-    if (needsOpenAiDictation) {
-      const transcriptionModel =
-        process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe";
-
-      dictationSttService = {
-        id: "openai",
-        createSession: ({ logger: sessionLogger, language, prompt }) =>
-          new OpenAIRealtimeTranscriptionSession({
-            apiKey: openaiDictationApiKey!,
-            logger: sessionLogger,
-            transcriptionModel,
-            ...(language ? { language } : {}),
-            ...(prompt ? { prompt } : {}),
-            turnDetection: null,
-          }),
-      };
-    }
-  } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
-    logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation is unavailable");
-  }
-
-  const effectiveProviders = {
-    dictationStt: dictationSttService?.id ?? "unavailable",
-    voiceStt: sttService?.id ?? "unavailable",
-    voiceTts: !ttsService ? "unavailable" : ttsService === sherpaTts ? "local" : "openai",
-  };
-  const unavailableFeatures = [
-    !dictationSttService ? "dictation.stt" : null,
-    !sttService ? "voice.stt" : null,
-    !ttsService ? "voice.tts" : null,
-  ].filter((feature): feature is string => feature !== null);
-
-  if (unavailableFeatures.length > 0) {
-    logger.error(
-      {
-        requestedProviders: {
-          dictationStt: dictationSttProvider,
-          voiceStt: voiceSttProvider,
-          voiceTts: voiceTtsProvider,
-        },
-        effectiveProviders,
-        unavailableFeatures,
-      },
-      "Speech provider reconciliation failed: configured features are unavailable"
-    );
-    throw new Error(
-      `Configured speech features unavailable: ${unavailableFeatures.join(", ")}`
-    );
-  } else {
-    logger.info(
-      {
-        effectiveProviders,
-      },
-      "Speech provider reconciliation completed"
-    );
-  }
-
-  const wsServer = new VoiceAssistantWebSocketServer(
+  wsServer = new VoiceAssistantWebSocketServer(
     httpServer,
     logger,
     serverId,
@@ -805,12 +565,26 @@ export async function createPaseoDaemon(
     { stt: sttService, tts: ttsService },
     terminalManager,
     {
-      openrouterApiKey: config.openrouterApiKey ?? null,
-      voiceLlmModel: config.voiceLlmModel ?? null,
+      voiceLlmProvider: resolvedVoiceLlmProvider,
+      voiceLlmModeId: resolvedVoiceLlmModeId,
+      voiceLlmProviderExplicit,
+      voiceLlmModel: resolvedVoiceLlmModel,
+      voiceAgentMcpStdio: {
+        command: voiceMcpBridgeCommand.command,
+        baseArgs: [
+          ...voiceMcpBridgeCommand.baseArgs,
+          "--socket",
+          voiceMcpSocketPath,
+        ],
+        env: {
+          PASEO_HOME: config.paseoHome,
+        },
+      },
     },
     {
       finalTimeoutMs: config.dictationFinalTimeoutMs,
       stt: dictationSttService,
+      localModels: localModelConfig ?? undefined,
     }
   );
 
@@ -870,7 +644,12 @@ export async function createPaseoDaemon(
               relayTransport?.stop().catch(() => undefined);
               relayTransport = startRelayTransport({
                 logger,
-                attachSocket: (ws) => wsServer.attachExternalSocket(ws),
+                attachSocket: (ws) => {
+                  if (!wsServer) {
+                    throw new Error("WebSocket server not initialized");
+                  }
+                  return wsServer.attachExternalSocket(ws);
+                },
                 relayEndpoint,
                 serverId,
                 daemonKeyPair: daemonKeyPair.keyPair,
@@ -896,6 +675,9 @@ export async function createPaseoDaemon(
         httpServer.listen(listenTarget.path);
       }
     });
+    if (voiceMcpBridgeServer) {
+      await voiceMcpBridgeServer.start();
+    }
   };
 
   const stop = async () => {
@@ -905,13 +687,14 @@ export async function createPaseoDaemon(
     await agentStorage.flush().catch(() => undefined);
     await shutdownProviders(logger);
     terminalManager.killAll();
-    if (sherpaTts && typeof (sherpaTts as any).free === "function") {
-      (sherpaTts as any).free();
-    }
-    sherpaOnline?.free();
-    sherpaOffline?.free();
+    cleanupSpeechRuntime();
     await relayTransport?.stop().catch(() => undefined);
-    await wsServer.close();
+    if (wsServer) {
+      await wsServer.close();
+    }
+    if (voiceMcpBridgeServer) {
+      await voiceMcpBridgeServer.stop().catch(() => undefined);
+    }
     await new Promise<void>((resolve) => {
       httpServer.close(() => resolve());
     });

packages/server/src/server/config.ts

@@ -1,9 +1,11 @@
 import path from "node:path";
+import { z } from "zod";
 
 import type { PaseoDaemonConfig } from "./bootstrap.js";
-import type { STTConfig } from "./speech/providers/openai/stt.js";
-import type { TTSConfig } from "./speech/providers/openai/tts.js";
 import { loadPersistedConfig } from "./persisted-config.js";
+import type { AgentProvider } from "./agent/agent-sdk-types.js";
+import { AgentProviderSchema } from "./agent/provider-manifest.js";
+import { resolveSpeechConfig } from "./speech/speech-config-resolver.js";
 import {
   mergeAllowedHosts,
   parseAllowedHostsEnv,
@@ -13,7 +15,6 @@ import {
 const DEFAULT_PORT = 6767;
 const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443";
 const DEFAULT_APP_BASE_URL = "https://app.paseo.sh";
-
 function getDefaultListen(): string {
   // Main HTTP server defaults to TCP
   return `127.0.0.1:${DEFAULT_PORT}`;
@@ -26,97 +27,14 @@ export type CliConfigOverrides = Partial<{
   allowedHosts: AllowedHostsConfig;
 }>;
 
-function parseOpenAIConfig(
-  env: NodeJS.ProcessEnv,
-  configApiKey: string | undefined,
-  config: {
-    dictationSttModel?: string;
-    dictationSttConfidenceThreshold?: number;
-    voiceSttModel?: string;
-    voiceTtsVoice?: TTSConfig["voice"];
-    voiceTtsModel?: TTSConfig["model"];
-  }
-) {
-  const apiKey = env.OPENAI_API_KEY ?? configApiKey;
-  if (!apiKey) return undefined;
+const OptionalVoiceLlmProviderSchema = z
+  .union([z.string(), z.null(), z.undefined()])
+  .transform((value): string | null => (typeof value === "string" ? value.trim().toLowerCase() : null))
+  .pipe(z.union([AgentProviderSchema, z.null()]));
 
-  const sttConfidenceThreshold = env.STT_CONFIDENCE_THRESHOLD
-    ? parseFloat(env.STT_CONFIDENCE_THRESHOLD)
-    : config.dictationSttConfidenceThreshold;
-  const sttModel = (
-    env.STT_MODEL ??
-    config.voiceSttModel ??
-    config.dictationSttModel
-  ) as STTConfig["model"] | undefined;
-  const ttsVoice = (env.TTS_VOICE || "alloy") as
-    | "alloy"
-    | "echo"
-    | "fable"
-    | "onyx"
-    | "nova"
-    | "shimmer";
-  const ttsModel = (env.TTS_MODEL || config.voiceTtsModel || "tts-1") as
-    | "tts-1"
-    | "tts-1-hd";
-  const configuredVoice = config.voiceTtsVoice;
-
-  return {
-    apiKey,
-    stt: {
-      apiKey,
-      confidenceThreshold: sttConfidenceThreshold,
-      ...(sttModel ? { model: sttModel } : {}),
-    },
-    tts: {
-      apiKey,
-      voice: configuredVoice ?? ttsVoice,
-      model: ttsModel,
-      responseFormat: "pcm" as TTSConfig["responseFormat"],
-    },
-  };
-}
-
-function parseSpeechProviderId(value: unknown): "openai" | "local" | null {
-  if (typeof value !== "string") {
-    return null;
-  }
-  const normalized = value.trim().toLowerCase();
-  if (!normalized) {
-    return null;
-  }
-  if (normalized === "openai") return "openai";
-  if (normalized === "local") return "local";
-  return null;
-}
-
-function normalizeSherpaSttPreset(value: string): string {
-  const raw = value.trim();
-  const normalized = raw.toLowerCase();
-  if (normalized === "zipformer" || normalized === "zipformer-bilingual") {
-    return "zipformer-bilingual-zh-en-2023-02-20";
-  }
-  if (normalized === "paraformer") {
-    return "paraformer-bilingual-zh-en";
-  }
-  if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") {
-    return "parakeet-tdt-0.6b-v3-int8";
-  }
-  return raw;
-}
-
-function normalizeSherpaTtsPreset(value: string): string {
-  const raw = value.trim();
-  const normalized = raw.toLowerCase();
-  if (normalized === "pocket" || normalized === "pocket-tts") {
-    return "pocket-tts-onnx-int8";
-  }
-  if (normalized === "kitten") {
-    return "kitten-nano-en-v0_1-fp16";
-  }
-  if (normalized === "kokoro") {
-    return "kokoro-en-v0_19";
-  }
-  return raw;
+function parseOptionalVoiceLlmProvider(value: unknown): AgentProvider | null {
+  const parsed = OptionalVoiceLlmProviderSchema.safeParse(value);
+  return parsed.success ? parsed.data : null;
 }
 
 export function loadConfig(
@@ -171,81 +89,19 @@ export function loadConfig(
   const appBaseUrl =
     env.PASEO_APP_BASE_URL ?? persisted.app?.baseUrl ?? DEFAULT_APP_BASE_URL;
 
-  const openai = parseOpenAIConfig(env, persisted.providers?.openai?.apiKey, {
-    dictationSttModel: persisted.features?.dictation?.stt?.model,
-    dictationSttConfidenceThreshold:
-      persisted.features?.dictation?.stt?.confidenceThreshold,
-    voiceSttModel: persisted.features?.voiceMode?.stt?.model,
-    voiceTtsModel: persisted.features?.voiceMode?.tts?.model,
-    voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice,
+  const { openai, speech } = resolveSpeechConfig({
+    paseoHome,
+    env,
+    persisted,
   });
 
-  const dictationSttProvider =
-    parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ??
-    parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ??
-    "local";
-
-  const voiceSttProvider =
-    parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ??
-    parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ??
-    "local";
-
-  const voiceTtsProvider =
-    parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ??
-    parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ??
-    "local";
-
-  const shouldConfigureSherpa =
-    dictationSttProvider === "local" ||
-    voiceSttProvider === "local" ||
-    voiceTtsProvider === "local" ||
-    typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" ||
-    Boolean(persisted.providers?.sherpaOnnx);
-
-  const sherpaModelsDir =
-    (env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() ||
-    path.join(paseoHome, "models", "sherpa-onnx");
-
-  const sherpaOnnx = shouldConfigureSherpa
-    ? {
-        modelsDir: sherpaModelsDir,
-        autoDownload:
-          env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined
-            ? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1"
-            : persisted.providers?.sherpaOnnx?.autoDownload ??
-              // In tests we should never hit the network unexpectedly.
-              Boolean(env.VITEST) === false,
-        stt: {
-          preset: normalizeSherpaSttPreset(
-            (env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() ||
-              (persisted.features?.voiceMode?.stt?.preset ??
-                persisted.features?.dictation?.stt?.preset)?.trim() ||
-              "parakeet-tdt-0.6b-v3-int8"
-          ),
-        },
-        tts: {
-          preset: normalizeSherpaTtsPreset(
-            (env.PASEO_SHERPA_TTS_PRESET ??
-              persisted.providers?.sherpaOnnx?.tts?.preset ??
-              persisted.features?.voiceMode?.tts?.preset)?.trim() ||
-              (env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8")
-          ),
-          speakerId:
-            env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined
-              ? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10)
-              : persisted.providers?.sherpaOnnx?.tts?.speakerId ??
-                persisted.features?.voiceMode?.tts?.speakerId,
-          speed:
-            env.PASEO_SHERPA_TTS_SPEED !== undefined
-              ? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED)
-              : persisted.providers?.sherpaOnnx?.tts?.speed ??
-                persisted.features?.voiceMode?.tts?.speed,
-        },
-      }
-    : undefined;
-
-  const openrouterApiKey =
-    env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null;
+  const envVoiceLlmProvider = parseOptionalVoiceLlmProvider(env.PASEO_VOICE_LLM_PROVIDER);
+  const persistedVoiceLlmProvider = parseOptionalVoiceLlmProvider(
+    persisted.features?.voiceMode?.llm?.provider
+  );
+  const voiceLlmProvider = envVoiceLlmProvider ?? persistedVoiceLlmProvider ?? null;
+  const voiceLlmProviderExplicit =
+    envVoiceLlmProvider !== null || persistedVoiceLlmProvider !== null;
   const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null;
 
   return {
@@ -265,13 +121,9 @@ export function loadConfig(
     relayPublicEndpoint,
     appBaseUrl,
     openai,
-    speech: {
-      dictationSttProvider,
-      voiceSttProvider,
-      voiceTtsProvider,
-      ...(sherpaOnnx ? { sherpaOnnx } : {}),
-    },
-    openrouterApiKey,
+    speech,
+    voiceLlmProvider,
+    voiceLlmProviderExplicit,
     voiceLlmModel,
   };
 }

packages/server/src/server/daemon-client.e2e.test.ts

@@ -3,6 +3,7 @@ import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs";
 import { tmpdir, homedir } from "node:os";
 import path from "node:path";
 import { execSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
 
 import {
   createDaemonTestContext,
@@ -18,9 +19,9 @@ import {
 
 const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
 
-const sherpaModelsDir =
-  process.env.PASEO_SHERPA_ONNX_MODELS_DIR ??
-  path.join(homedir(), ".paseo", "models", "sherpa-onnx");
+const localModelsDir =
+  process.env.PASEO_LOCAL_MODELS_DIR ??
+  path.join(homedir(), ".paseo", "models", "local-speech");
 
 function hasSherpaZipformerModels(modelsDir: string): boolean {
   return (
@@ -49,7 +50,7 @@ function hasSherpaKittenModels(modelsDir: string): boolean {
   );
 }
 
-const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir);
+const hasLocalSpeech = hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir);
 const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey);
 const speechTest = hasAnySpeech ? test : test.skip;
 
@@ -92,23 +93,42 @@ describe("daemon client E2E", () => {
   let ctx: DaemonTestContext;
 
   beforeAll(async () => {
+    const speechConfig =
+      openaiApiKey
+        ? {
+            providers: {
+              dictationStt: { provider: "openai" as const, explicit: true },
+              voiceStt: { provider: "openai" as const, explicit: true },
+              voiceTts: { provider: "openai" as const, explicit: true },
+            },
+          }
+        : hasLocalSpeech
+          ? {
+              providers: {
+                dictationStt: { provider: "local" as const, explicit: true },
+                voiceStt: { provider: "local" as const, explicit: true },
+                voiceTts: { provider: "local" as const, explicit: true },
+              },
+              local: {
+                modelsDir: localModelsDir,
+                models: {
+                  dictationStt:
+                    process.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
+                    "zipformer-bilingual-zh-en-2023-02-20",
+                  voiceStt:
+                    process.env.PASEO_VOICE_LOCAL_STT_MODEL ??
+                    "zipformer-bilingual-zh-en-2023-02-20",
+                  voiceTts:
+                    process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
+                },
+              },
+            }
+          : undefined;
+
     ctx = await createDaemonTestContext({
       dictationFinalTimeoutMs: 5000,
       ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}),
-      speech: {
-        dictationSttProvider: "local",
-        voiceSttProvider: "local",
-        voiceTtsProvider: "local",
-        sherpaOnnx: {
-          modelsDir: sherpaModelsDir,
-          stt: {
-            preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20",
-          },
-          tts: {
-            preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16",
-          },
-        },
-      },
+      ...(speechConfig ? { speech: speechConfig } : {}),
     });
   }, 60000);
 
@@ -119,22 +139,18 @@ describe("daemon client E2E", () => {
   test("handles session actions", async () => {
     expect(ctx.client.isConnected).toBe(true);
 
-    const voiceConversationId = `voice-${Date.now()}`;
-    const loadResult = await ctx.client.loadVoiceConversation(voiceConversationId);
-    expect(loadResult.voiceConversationId).toBe(voiceConversationId);
-    expect(typeof loadResult.messageCount).toBe("number");
-
     const agents = await ctx.client.fetchAgents();
     expect(Array.isArray(agents)).toBe(true);
 
-    const listResult = await ctx.client.listVoiceConversations();
-    expect(Array.isArray(listResult.conversations)).toBe(true);
+    const voiceAgents = await ctx.client.fetchAgents({
+      filter: { labels: { surface: "voice" } },
+    });
+    expect(Array.isArray(voiceAgents)).toBe(true);
 
-    const missingId = `missing-${Date.now()}`;
-    const deleteResult = await ctx.client.deleteVoiceConversation(missingId);
-    expect(deleteResult.voiceConversationId).toBe(missingId);
-    expect(deleteResult.success).toBe(false);
-    expect(deleteResult.error).toBeTruthy();
+    await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined();
+    await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined();
+
+    await ctx.client.deleteAgent(randomUUID());
   }, 30000);
 
   test("emits server_info on websocket connect", async () => {
@@ -160,19 +176,23 @@ describe("daemon client E2E", () => {
     await client.close();
   }, 15000);
 
-  test("matches request IDs for concurrent session requests", async () => {
-    const firstRequestId = `list-${Date.now()}-a`;
-    const secondRequestId = `list-${Date.now()}-b`;
+  test("handles concurrent filtered agent fetch requests", async () => {
+    const firstRequestId = `fetch-${Date.now()}-a`;
+    const secondRequestId = `fetch-${Date.now()}-b`;
 
     const [first, second] = await Promise.all([
-      ctx.client.listVoiceConversations(firstRequestId),
-      ctx.client.listVoiceConversations(secondRequestId),
+      ctx.client.fetchAgents({
+        requestId: firstRequestId,
+        filter: { labels: { surface: "voice" } },
+      }),
+      ctx.client.fetchAgents({
+        requestId: secondRequestId,
+        filter: { labels: { surface: "voice" } },
+      }),
     ]);
 
-    expect(Array.isArray(first.conversations)).toBe(true);
-    expect(Array.isArray(second.conversations)).toBe(true);
-    expect(first.requestId).toBe(firstRequestId);
-    expect(second.requestId).toBe(secondRequestId);
+    expect(Array.isArray(first)).toBe(true);
+    expect(Array.isArray(second)).toBe(true);
   }, 15000);
 
   test(
@@ -355,7 +375,7 @@ describe("daemon client E2E", () => {
       expect(sawAssistantMessage).toBe(true);
       expect(sawRawAssistantMessage).toBe(true);
 
-      await ctx.client.setVoiceConversation(false);
+      await ctx.client.setVoiceMode(false);
 
       await ctx.client.abortRequest();
       await ctx.client.audioPlayed("audio-1");
@@ -566,16 +586,16 @@ describe("daemon client E2E", () => {
     120000
   );
 
-  test.runIf(Boolean(process.env.OPENROUTER_API_KEY))(
-    "streams session activity logs and chunks",
+  speechTest(
+    "does not process non-voice audio through the voice agent path",
     async () => {
-      await ctx.client.setVoiceConversation(false);
+      await ctx.client.setVoiceMode(false);
 
-      let sawAssistantChunk = false;
       let sawTranscriptLog = false;
+      let sawAssistantChunk = false;
       let sawAssistantLog = false;
 
-      const completion = waitForSignal(60000, (resolve) => {
+      const transcriptSeen = waitForSignal(60000, (resolve) => {
         const unsubscribeChunk = ctx.client.on("assistant_chunk", (message) => {
           if (message.type !== "assistant_chunk") {
             return;
@@ -583,9 +603,6 @@ describe("daemon client E2E", () => {
           if (message.payload.chunk.length > 0) {
             sawAssistantChunk = true;
           }
-          if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) {
-            resolve();
-          }
         });
 
         const unsubscribeActivity = ctx.client.on("activity_log", (message) => {
@@ -594,13 +611,11 @@ describe("daemon client E2E", () => {
           }
           if (message.payload.type === "transcript") {
             sawTranscriptLog = true;
+            resolve();
           }
           if (message.payload.type === "assistant") {
             sawAssistantLog = true;
           }
-          if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) {
-            resolve();
-          }
         });
 
         return () => {
@@ -609,16 +624,30 @@ describe("daemon client E2E", () => {
         };
       });
 
-      await ctx.client.sendUserMessage("Say 'hello' and nothing else");
-      await completion;
+      const fixturePath = path.resolve(
+        process.cwd(),
+        "..",
+        "app",
+        "e2e",
+        "fixtures",
+        "recording.wav"
+      );
+      const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
+      await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true);
+      await transcriptSeen;
+      await new Promise((resolve) => setTimeout(resolve, 1500));
+
+      expect(sawTranscriptLog).toBe(true);
+      expect(sawAssistantChunk).toBe(false);
+      expect(sawAssistantLog).toBe(false);
     },
-    120000
+    90000
   );
 
   speechTest(
     "voice mode buffers audio until isLast and emits transcription_result",
     async () => {
-      await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`);
+      await ctx.client.setVoiceMode(true, randomUUID());
 
       const transcription = waitForSignal(30_000, (resolve) => {
         const unsubscribe = ctx.client.on("transcription_result", (message) => {
@@ -713,7 +742,7 @@ describe("daemon client E2E", () => {
         }
       } finally {
         await Promise.allSettled([transcription, errorSignal]);
-        await ctx.client.setVoiceConversation(false);
+        await ctx.client.setVoiceMode(false);
       }
     },
     90_000

packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts

@@ -186,6 +186,7 @@ describe("ConnectionOfferV2 (daemon E2E)", () => {
         PASEO_HOME: tempHome,
         PASEO_LISTEN: `0.0.0.0:${port}`,
         OPENAI_API_KEY: "",
+        PASEO_LOCAL_AUTO_DOWNLOAD: "0",
         PASEO_LOG_FORMAT: "json",
       };
 

packages/server/src/server/exports.ts

@@ -5,6 +5,7 @@ export { resolvePaseoHome } from "./paseo-home.js";
 export { createRootLogger, type LogLevel, type LogFormat } from "./logger.js";
 export { loadPersistedConfig, type PersistedConfig } from "./persisted-config.js";
 export { DaemonClient, type DaemonClientConfig, type ConnectionState, type DaemonEvent } from "../client/daemon-client.js";
+export { runVoiceMcpBridgeCli } from "./voice-mcp-bridge.js";
 
 // Agent SDK types for CLI commands
 export type {

packages/server/src/server/index.ts

@@ -10,8 +10,15 @@ import { resolvePaseoHome } from "./paseo-home.js";
 import { createRootLogger } from "./logger.js";
 import { loadPersistedConfig } from "./persisted-config.js";
 import { PidLockError } from "./pid-lock.js";
+import { runVoiceMcpBridgeCli } from "./voice-mcp-bridge.js";
 
 async function main() {
+  const bridgeArgIndex = process.argv.findIndex((arg) => arg === "__paseo_voice_mcp_bridge");
+  if (bridgeArgIndex >= 0) {
+    await runVoiceMcpBridgeCli(process.argv.slice(bridgeArgIndex + 1));
+    return;
+  }
+
   let paseoHome: string;
   let logger: ReturnType<typeof createRootLogger>;
   let config: ReturnType<typeof loadConfig>;

packages/server/src/server/path-utils.ts

@@ -0,0 +1,22 @@
+import { homedir } from "node:os";
+import { isAbsolute, resolve } from "node:path";
+
+function hasHomePrefix(value: string): boolean {
+  return value === "~" || value.startsWith("~/");
+}
+
+export function expandUserPath(value: string): string {
+  const trimmed = value.trim();
+  if (hasHomePrefix(trimmed)) {
+    return resolve(homedir(), trimmed.slice(2));
+  }
+  return resolve(trimmed);
+}
+
+export function resolvePathFromBase(baseCwd: string, requestedPath: string): string {
+  const trimmed = requestedPath.trim();
+  if (hasHomePrefix(trimmed) || isAbsolute(trimmed)) {
+    return expandUserPath(trimmed);
+  }
+  return resolve(baseCwd, trimmed);
+}

packages/server/src/server/persisted-config.ts

@@ -1,6 +1,7 @@
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import path from "node:path";
 import { z } from "zod";
+import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
 
 const LogConfigSchema = z
   .object({
@@ -17,44 +18,25 @@ const ProviderCredentialsSchema = z
   })
   .strict();
 
-const SherpaOnnxProviderSchema = z
+const LocalSpeechProviderSchema = z
   .object({
     modelsDir: z.string().min(1).optional(),
     autoDownload: z.boolean().optional(),
-    stt: z
-      .object({
-        preset: z.string().min(1).optional(),
-      })
-      .strict()
-      .optional(),
-    tts: z
-      .object({
-        preset: z.string().min(1).optional(),
-        speakerId: z.number().int().optional(),
-        speed: z.number().optional(),
-      })
-      .strict()
-      .optional(),
   })
   .strict();
 
 const ProvidersSchema = z
   .object({
     openai: ProviderCredentialsSchema.optional(),
-    openrouter: ProviderCredentialsSchema.optional(),
-    sherpaOnnx: SherpaOnnxProviderSchema.optional(),
+    local: LocalSpeechProviderSchema.optional(),
   })
   .strict();
 
-const SpeechProviderIdSchema = z.preprocess(
-  (value) => {
-    if (typeof value !== "string") {
-      return value;
-    }
-    return value.trim().toLowerCase();
-  },
-  z.enum(["openai", "local"])
-);
+const SpeechProviderIdSchema = z
+  .string()
+  .trim()
+  .toLowerCase()
+  .pipe(z.enum(["openai", "local"]));
 
 const FeatureDictationSchema = z
   .object({
@@ -62,7 +44,6 @@ const FeatureDictationSchema = z
       .object({
         provider: SpeechProviderIdSchema.optional(),
         model: z.string().min(1).optional(),
-        preset: z.string().min(1).optional(),
         confidenceThreshold: z.number().optional(),
       })
       .strict()
@@ -74,7 +55,7 @@ const FeatureVoiceModeSchema = z
   .object({
     llm: z
       .object({
-        provider: z.enum(["openrouter"]).optional(),
+        provider: z.enum(AGENT_PROVIDER_IDS as [string, ...string[]]).optional(),
         model: z.string().min(1).optional(),
       })
       .strict()
@@ -83,16 +64,14 @@ const FeatureVoiceModeSchema = z
       .object({
         provider: SpeechProviderIdSchema.optional(),
         model: z.string().min(1).optional(),
-        preset: z.string().min(1).optional(),
       })
       .strict()
       .optional(),
     tts: z
       .object({
         provider: SpeechProviderIdSchema.optional(),
-        model: z.enum(["tts-1", "tts-1-hd"]).optional(),
+        model: z.string().min(1).optional(),
         voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(),
-        preset: z.string().min(1).optional(),
         speakerId: z.number().int().optional(),
         speed: z.number().optional(),
       })

packages/server/src/server/session.voice-mcp-config.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, test } from "vitest";
+
+import { buildVoiceAgentMcpServerConfig } from "./session.js";
+
+describe("voice MCP stdio config", () => {
+  test("builds stdio MCP config for voice agent", () => {
+    const config = buildVoiceAgentMcpServerConfig({
+      callerAgentId: "voice-agent-123",
+      command: "/usr/local/bin/paseo",
+      baseArgs: ["__paseo_voice_mcp_bridge", "--socket", "/tmp/paseo-voice.sock"],
+      env: {
+        PASEO_HOME: "/tmp/paseo-home",
+      },
+    });
+
+    expect(config.type).toBe("stdio");
+    expect(config.command).toBe("/usr/local/bin/paseo");
+    expect(config.args).toEqual([
+      "__paseo_voice_mcp_bridge",
+      "--socket",
+      "/tmp/paseo-voice.sock",
+      "--caller-agent-id",
+      "voice-agent-123",
+    ]);
+    expect(config.env).toEqual({
+      PASEO_HOME: "/tmp/paseo-home",
+    });
+  });
+});

packages/server/src/server/speech/providers/local/config.ts

@@ -0,0 +1,167 @@
+import path from "node:path";
+
+import { z } from "zod";
+
+import type { PersistedConfig } from "../../../persisted-config.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import {
+  DEFAULT_LOCAL_STT_MODEL,
+  DEFAULT_LOCAL_TTS_MODEL,
+  LocalSttModelIdSchema,
+  LocalTtsModelIdSchema,
+  type LocalSpeechModelId,
+  type LocalSttModelId,
+  type LocalTtsModelId,
+} from "./models.js";
+
+export type LocalSpeechModelConfig = {
+  dictationStt: LocalSttModelId;
+  voiceStt: LocalSttModelId;
+  voiceTts: LocalTtsModelId;
+  voiceTtsSpeakerId?: number;
+  voiceTtsSpeed?: number;
+};
+
+export type LocalSpeechProviderConfig = {
+  modelsDir: string;
+  autoDownload?: boolean;
+  models: LocalSpeechModelConfig;
+};
+
+export type ResolvedLocalSpeechConfig = {
+  local: LocalSpeechProviderConfig | undefined;
+};
+
+export type { LocalSpeechModelId, LocalSttModelId, LocalTtsModelId };
+
+const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
+
+const BooleanStringSchema = z
+  .string()
+  .trim()
+  .toLowerCase()
+  .pipe(z.enum(["1", "0", "true", "false", "yes", "no"]))
+  .transform((value) => value === "1" || value === "true" || value === "yes");
+
+const OptionalBooleanFlagSchema = z
+  .union([z.boolean(), BooleanStringSchema])
+  .optional();
+
+const NumberLikeSchema = z.union([
+  z.number(),
+  z.string().trim().min(1),
+]);
+
+const OptionalFiniteNumberSchema = NumberLikeSchema
+  .pipe(z.coerce.number().finite())
+  .optional();
+
+const OptionalIntegerSchema = NumberLikeSchema
+  .pipe(z.coerce.number().int())
+  .optional();
+
+const LocalSpeechResolutionSchema = z.object({
+  includeProviderConfig: z.boolean(),
+  modelsDir: z.string().trim().min(1),
+  autoDownload: OptionalBooleanFlagSchema.default(true),
+  dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
+  voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
+  voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL),
+  voiceLocalTtsSpeakerId: OptionalIntegerSchema,
+  voiceLocalTtsSpeed: OptionalFiniteNumberSchema,
+});
+
+function persistedLocalFeatureModel(
+  provider: RequestedSpeechProviders[keyof RequestedSpeechProviders]["provider"],
+  model: string | undefined
+): string | undefined {
+  if (provider !== "local") {
+    return undefined;
+  }
+  return model;
+}
+
+function shouldIncludeLocalProviderConfig(params: {
+  providers: RequestedSpeechProviders;
+  env: NodeJS.ProcessEnv;
+  persisted: PersistedConfig;
+}): boolean {
+  const localRequestedByFeature =
+    params.providers.dictationStt.provider === "local" ||
+    params.providers.voiceStt.provider === "local" ||
+    params.providers.voiceTts.provider === "local";
+
+  return (
+    localRequestedByFeature ||
+    params.env.PASEO_LOCAL_MODELS_DIR !== undefined ||
+    params.persisted.providers?.local !== undefined
+  );
+}
+
+export function resolveLocalSpeechConfig(params: {
+  paseoHome: string;
+  env: NodeJS.ProcessEnv;
+  persisted: PersistedConfig;
+  providers: RequestedSpeechProviders;
+}): ResolvedLocalSpeechConfig {
+  const includeProviderConfig = shouldIncludeLocalProviderConfig(params);
+
+  const parsed = LocalSpeechResolutionSchema.parse({
+    includeProviderConfig,
+    modelsDir:
+      params.env.PASEO_LOCAL_MODELS_DIR ??
+      params.persisted.providers?.local?.modelsDir ??
+      path.join(params.paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR),
+    autoDownload:
+      params.env.PASEO_LOCAL_AUTO_DOWNLOAD ??
+      params.persisted.providers?.local?.autoDownload,
+    dictationLocalSttModel:
+      params.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
+      persistedLocalFeatureModel(
+        params.providers.dictationStt.provider,
+        params.persisted.features?.dictation?.stt?.model
+      ) ??
+      DEFAULT_LOCAL_STT_MODEL,
+    voiceLocalSttModel:
+      params.env.PASEO_VOICE_LOCAL_STT_MODEL ??
+      persistedLocalFeatureModel(
+        params.providers.voiceStt.provider,
+        params.persisted.features?.voiceMode?.stt?.model
+      ) ??
+      DEFAULT_LOCAL_STT_MODEL,
+    voiceLocalTtsModel:
+      params.env.PASEO_VOICE_LOCAL_TTS_MODEL ??
+      persistedLocalFeatureModel(
+        params.providers.voiceTts.provider,
+        params.persisted.features?.voiceMode?.tts?.model
+      ) ??
+      DEFAULT_LOCAL_TTS_MODEL,
+    voiceLocalTtsSpeakerId:
+      params.env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID ??
+      params.persisted.features?.voiceMode?.tts?.speakerId,
+    voiceLocalTtsSpeed:
+      params.env.PASEO_VOICE_LOCAL_TTS_SPEED ??
+      params.persisted.features?.voiceMode?.tts?.speed,
+  });
+
+  return {
+    local:
+      parsed.includeProviderConfig
+        ? {
+            modelsDir: parsed.modelsDir,
+            autoDownload: parsed.autoDownload,
+            models: {
+              dictationStt: parsed.dictationLocalSttModel,
+              voiceStt: parsed.voiceLocalSttModel,
+              voiceTts: parsed.voiceLocalTtsModel,
+              ...(parsed.voiceLocalTtsSpeakerId !== undefined
+                ? { voiceTtsSpeakerId: parsed.voiceLocalTtsSpeakerId }
+                : {}),
+              ...(parsed.voiceLocalTtsSpeed !== undefined
+                ? { voiceTtsSpeed: parsed.voiceLocalTtsSpeed }
+                : {}),
+            },
+          }
+        : undefined,
+  };
+}

packages/server/src/server/speech/providers/local/models.ts

@@ -0,0 +1,48 @@
+import {
+  ensureSherpaOnnxModels,
+  getSherpaOnnxModelDir,
+} from "./sherpa/model-downloader.js";
+import {
+  DEFAULT_LOCAL_STT_MODEL,
+  DEFAULT_LOCAL_TTS_MODEL,
+  LocalSttModelIdSchema,
+  LocalTtsModelIdSchema,
+  listSherpaOnnxModels,
+  type LocalSpeechModelId,
+  type LocalSttModelId,
+  type LocalTtsModelId,
+} from "./sherpa/model-catalog.js";
+
+export {
+  DEFAULT_LOCAL_STT_MODEL,
+  DEFAULT_LOCAL_TTS_MODEL,
+  LocalSttModelIdSchema,
+  LocalTtsModelIdSchema,
+  type LocalSpeechModelId,
+  type LocalSttModelId,
+  type LocalTtsModelId,
+};
+
+export type LocalSpeechModelSpec = ReturnType<typeof listSherpaOnnxModels>[number];
+
+export function listLocalSpeechModels(): LocalSpeechModelSpec[] {
+  return listSherpaOnnxModels();
+}
+
+export function getLocalSpeechModelDir(modelsDir: string, modelId: LocalSpeechModelId): string {
+  return getSherpaOnnxModelDir(modelsDir, modelId);
+}
+
+export async function ensureLocalSpeechModels(options: {
+  modelsDir: string;
+  modelIds: LocalSpeechModelId[];
+  autoDownload?: boolean;
+  logger: import("pino").Logger;
+}): Promise<Record<LocalSpeechModelId, string>> {
+  return ensureSherpaOnnxModels({
+    modelsDir: options.modelsDir,
+    modelIds: options.modelIds,
+    autoDownload: options.autoDownload ?? true,
+    logger: options.logger,
+  });
+}

packages/server/src/server/speech/providers/local/runtime.ts

@@ -0,0 +1,370 @@
+import type { Logger } from "pino";
+
+import type { PaseoSpeechConfig } from "../../../bootstrap.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js";
+import {
+  ensureLocalSpeechModels,
+  getLocalSpeechModelDir,
+  DEFAULT_LOCAL_STT_MODEL,
+  DEFAULT_LOCAL_TTS_MODEL,
+  LocalSttModelIdSchema,
+  LocalTtsModelIdSchema,
+  type LocalSpeechModelId,
+  type LocalSttModelId,
+  type LocalTtsModelId,
+} from "./models.js";
+import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
+import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js";
+import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
+import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
+import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js";
+import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js";
+import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
+
+type LocalSttEngine =
+  | { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
+  | { kind: "online"; engine: SherpaOnlineRecognizerEngine };
+
+type ResolvedLocalModels = {
+  dictationLocalSttModel: LocalSttModelId;
+  voiceLocalSttModel: LocalSttModelId;
+  voiceLocalTtsModel: LocalTtsModelId;
+};
+
+type LocalSpeechAvailability = {
+  configured: boolean;
+  modelsDir: string | null;
+  autoDownload: boolean | null;
+};
+
+export type InitializedLocalSpeech = {
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+  dictationSttService: SpeechToTextProvider | null;
+  localVoiceTtsProvider: TextToSpeechProvider | null;
+  localModelConfig: {
+    modelsDir: string;
+    defaultModelIds: LocalSpeechModelId[];
+  } | null;
+  availability: LocalSpeechAvailability;
+  cleanup: () => void;
+};
+
+function buildModelDownloadHint(modelId: LocalSpeechModelId): string {
+  return `Use 'paseo speech download --model ${modelId}' to download this model.`;
+}
+
+function resolveConfiguredLocalModels(
+  speechConfig: PaseoSpeechConfig | null
+): ResolvedLocalModels {
+  return {
+    dictationLocalSttModel: LocalSttModelIdSchema.parse(
+      speechConfig?.local?.models.dictationStt ?? DEFAULT_LOCAL_STT_MODEL
+    ),
+    voiceLocalSttModel: LocalSttModelIdSchema.parse(
+      speechConfig?.local?.models.voiceStt ?? DEFAULT_LOCAL_STT_MODEL
+    ),
+    voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
+      speechConfig?.local?.models.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL
+    ),
+  };
+}
+
+export function getLocalSpeechAvailability(
+  speechConfig: PaseoSpeechConfig | null
+): LocalSpeechAvailability {
+  const localConfig = speechConfig?.local ?? null;
+  return {
+    configured: Boolean(localConfig),
+    modelsDir: localConfig?.modelsDir ?? null,
+    autoDownload: localConfig?.autoDownload ?? null,
+  };
+}
+
+function computeRequiredLocalModelIds(params: {
+  providers: RequestedSpeechProviders;
+  models: ResolvedLocalModels;
+}): LocalSpeechModelId[] {
+  const ids = new Set<LocalSpeechModelId>();
+  if (params.providers.dictationStt.provider === "local") {
+    ids.add(params.models.dictationLocalSttModel);
+  }
+  if (params.providers.voiceStt.provider === "local") {
+    ids.add(params.models.voiceLocalSttModel);
+  }
+  if (params.providers.voiceTts.provider === "local") {
+    ids.add(params.models.voiceLocalTtsModel);
+  }
+  return Array.from(ids);
+}
+
+async function createLocalSttEngine(params: {
+  modelId: LocalSttModelId;
+  modelsDir: string;
+  logger: Logger;
+}): Promise<LocalSttEngine> {
+  const { modelId, modelsDir, logger } = params;
+
+  if (modelId === "parakeet-tdt-0.6b-v3-int8") {
+    const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
+    return {
+      kind: "offline",
+      engine: new SherpaOfflineRecognizerEngine(
+        {
+          model: {
+            kind: "nemo_transducer",
+            encoder: `${modelDir}/encoder.int8.onnx`,
+            decoder: `${modelDir}/decoder.int8.onnx`,
+            joiner: `${modelDir}/joiner.int8.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+          },
+          numThreads: 2,
+          debug: 0,
+        },
+        logger
+      ),
+    };
+  }
+
+  if (modelId === "paraformer-bilingual-zh-en") {
+    const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
+    return {
+      kind: "online",
+      engine: new SherpaOnlineRecognizerEngine(
+        {
+          model: {
+            kind: "paraformer",
+            encoder: `${modelDir}/encoder.int8.onnx`,
+            decoder: `${modelDir}/decoder.int8.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+          },
+          numThreads: 1,
+          debug: 0,
+        },
+        logger
+      ),
+    };
+  }
+
+  if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
+    const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
+    return {
+      kind: "online",
+      engine: new SherpaOnlineRecognizerEngine(
+        {
+          model: {
+            kind: "transducer",
+            encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
+            decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
+            joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+            modelType: "zipformer",
+          },
+          numThreads: 1,
+          debug: 0,
+        },
+        logger
+      ),
+    };
+  }
+
+  throw new Error(`Unsupported local STT model '${modelId}'`);
+}
+
+export async function initializeLocalSpeechServices(params: {
+  providers: RequestedSpeechProviders;
+  speechConfig: PaseoSpeechConfig | null;
+  logger: Logger;
+}): Promise<InitializedLocalSpeech> {
+  const { providers, logger, speechConfig } = params;
+  const localConfig = speechConfig?.local ?? null;
+  const localModels = resolveConfiguredLocalModels(speechConfig);
+
+  let sttService: SpeechToTextProvider | null = null;
+  let ttsService: TextToSpeechProvider | null = null;
+  let dictationSttService: SpeechToTextProvider | null = null;
+  let localVoiceTtsProvider: TextToSpeechProvider | null = null;
+
+  const requiredLocalModelIds = computeRequiredLocalModelIds({
+    providers,
+    models: localModels,
+  });
+
+  if (requiredLocalModelIds.length > 0 && localConfig) {
+    try {
+      logger.info(
+        {
+          modelsDir: localConfig.modelsDir,
+          modelIds: requiredLocalModelIds,
+          autoDownload: localConfig.autoDownload ?? true,
+        },
+        "Ensuring local speech models"
+      );
+      await ensureLocalSpeechModels({
+        modelsDir: localConfig.modelsDir,
+        modelIds: requiredLocalModelIds,
+        autoDownload: localConfig.autoDownload ?? true,
+        logger,
+      });
+    } catch (err) {
+      logger.error(
+        {
+          err,
+          modelsDir: localConfig.modelsDir,
+          modelIds: requiredLocalModelIds,
+          autoDownload: localConfig.autoDownload ?? true,
+          hint:
+            "Use `paseo speech models` to inspect status and " +
+            "`paseo speech download --model <MODEL_ID>` to fetch missing models.",
+        },
+        "Failed to ensure local speech models"
+      );
+    }
+  }
+
+  const localSttEngines = new Map<LocalSttModelId, LocalSttEngine>();
+
+  const getLocalSttEngine = async (
+    modelId: LocalSttModelId
+  ): Promise<LocalSttEngine | null> => {
+    const existing = localSttEngines.get(modelId);
+    if (existing) {
+      return existing;
+    }
+    if (!localConfig) {
+      return null;
+    }
+    try {
+      const created = await createLocalSttEngine({
+        modelId,
+        modelsDir: localConfig.modelsDir,
+        logger,
+      });
+      localSttEngines.set(modelId, created);
+      return created;
+    } catch (err) {
+      logger.error(
+        {
+          err,
+          modelsDir: localConfig.modelsDir,
+          modelId,
+          hint: buildModelDownloadHint(modelId),
+        },
+        "Failed to initialize local STT engine (models missing or invalid)"
+      );
+      return null;
+    }
+  };
+
+  if (providers.voiceStt.provider === "local") {
+    if (!localConfig) {
+      logger.warn(
+        { configured: false },
+        "Local STT selected for voice but local provider config is missing; STT will be unavailable"
+      );
+    } else {
+      const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel);
+      if (voiceEngine?.kind === "offline") {
+        sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
+      } else if (voiceEngine?.kind === "online") {
+        sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
+      }
+    }
+  }
+
+  if (providers.dictationStt.provider === "local") {
+    if (!localConfig) {
+      logger.warn(
+        { configured: false },
+        "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable"
+      );
+    } else {
+      const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel);
+      if (dictationEngine?.kind === "offline") {
+        dictationSttService = {
+          id: "local",
+          createSession: () =>
+            new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
+        };
+      } else if (dictationEngine?.kind === "online") {
+        dictationSttService = {
+          id: "local",
+          createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
+        };
+      }
+    }
+  }
+
+  if (providers.voiceTts.provider === "local") {
+    if (!localConfig) {
+      logger.warn(
+        { configured: false },
+        "Local TTS selected for voice but local provider config is missing; TTS will be unavailable"
+      );
+    } else {
+      try {
+        if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
+          const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
+          localVoiceTtsProvider = await PocketTtsOnnxTTS.create(
+            {
+              modelDir,
+              precision: "int8",
+              targetChunkMs: 50,
+            },
+            logger
+          );
+        } else {
+          const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
+          localVoiceTtsProvider = new SherpaOnnxTTS(
+            {
+              preset: localModels.voiceLocalTtsModel,
+              modelDir,
+              speakerId: speechConfig?.local?.models.voiceTtsSpeakerId,
+              speed: speechConfig?.local?.models.voiceTtsSpeed,
+            },
+            logger
+          );
+        }
+        ttsService = localVoiceTtsProvider;
+      } catch (err) {
+        logger.error(
+          {
+            err,
+            modelsDir: localConfig.modelsDir,
+            modelId: localModels.voiceLocalTtsModel,
+            hint: buildModelDownloadHint(localModels.voiceLocalTtsModel),
+          },
+          "Failed to initialize local TTS engine (models missing or invalid)"
+        );
+      }
+    }
+  }
+
+  const cleanup = () => {
+    const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null;
+    if (typeof maybeFreeable?.free === "function") {
+      maybeFreeable.free();
+    }
+    for (const engine of localSttEngines.values()) {
+      engine.engine.free();
+    }
+  };
+
+  return {
+    sttService,
+    ttsService,
+    dictationSttService,
+    localVoiceTtsProvider,
+    localModelConfig:
+      localConfig
+        ? {
+            modelsDir: localConfig.modelsDir,
+            defaultModelIds: requiredLocalModelIds,
+          }
+        : null,
+    availability: getLocalSpeechAvailability(speechConfig),
+    cleanup,
+  };
+}

packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts

@@ -1,26 +1,22 @@
+import { z } from "zod";
+
 export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts";
 
-export type SherpaOnnxModelId =
-  | "zipformer-bilingual-zh-en-2023-02-20"
-  | "paraformer-bilingual-zh-en"
-  | "parakeet-tdt-0.6b-v3-int8"
-  | "kitten-nano-en-v0_1-fp16"
-  | "kokoro-en-v0_19"
-  | "pocket-tts-onnx-int8";
+type DefaultModelRole = "stt" | "tts";
 
-export type SherpaOnnxModelSpec = {
-  id: SherpaOnnxModelId;
+type SherpaOnnxCatalogEntry = {
   kind: SherpaOnnxModelKind;
   archiveUrl?: string;
   downloadFiles?: Array<{ url: string; relPath: string }>;
   extractedDir: string;
   requiredFiles: string[];
   description: string;
+  aliases?: readonly string[];
+  defaultFor?: DefaultModelRole;
 };
 
-export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxModelSpec> = {
+export const SHERPA_ONNX_MODEL_CATALOG = {
   "zipformer-bilingual-zh-en-2023-02-20": {
-    id: "zipformer-bilingual-zh-en-2023-02-20",
     kind: "stt-online",
     archiveUrl:
       "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2",
@@ -32,44 +28,45 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxMode
       "tokens.txt",
     ],
     description: "Streaming Zipformer transducer (fast, good accuracy).",
+    aliases: ["zipformer", "zipformer-bilingual"],
   },
   "paraformer-bilingual-zh-en": {
-    id: "paraformer-bilingual-zh-en",
     kind: "stt-online",
     archiveUrl:
       "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2",
     extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en",
     requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"],
     description: "Streaming Paraformer (often strong accuracy; heavier).",
+    aliases: ["paraformer"],
   },
   "parakeet-tdt-0.6b-v3-int8": {
-    id: "parakeet-tdt-0.6b-v3-int8",
     kind: "stt-offline",
     archiveUrl:
       "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
     extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
     requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
     description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).",
+    aliases: ["parakeet", "parakeet-v3", "parakeet-tdt"],
+    defaultFor: "stt",
   },
   "kitten-nano-en-v0_1-fp16": {
-    id: "kitten-nano-en-v0_1-fp16",
     kind: "tts",
     archiveUrl:
       "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2",
     extractedDir: "kitten-nano-en-v0_1-fp16",
     requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
     description: "KittenTTS (small, fast English TTS).",
+    aliases: ["kitten"],
   },
   "kokoro-en-v0_19": {
-    id: "kokoro-en-v0_19",
     kind: "tts",
     archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2",
     extractedDir: "kokoro-en-v0_19",
     requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
     description: "Kokoro TTS (higher quality; larger).",
+    aliases: ["kokoro"],
   },
   "pocket-tts-onnx-int8": {
-    id: "pocket-tts-onnx-int8",
     kind: "tts",
     extractedDir: "pocket-tts-onnx-int8",
     downloadFiles: [
@@ -112,17 +109,110 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxMode
       "reference_sample.wav",
     ],
     description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).",
+    aliases: ["pocket", "pocket-tts"],
+    defaultFor: "tts",
   },
+} as const satisfies Record<string, SherpaOnnxCatalogEntry>;
+
+export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG;
+export type LocalSpeechModelId = SherpaOnnxModelId;
+
+type ModelIdByKind<K extends SherpaOnnxModelKind> = {
+  [Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K
+    ? Id
+    : never;
+}[SherpaOnnxModelId];
+
+export type LocalSttModelId = ModelIdByKind<"stt-online"> | ModelIdByKind<"stt-offline">;
+export type LocalTtsModelId = ModelIdByKind<"tts">;
+
+const ALL_MODEL_IDS = Object.keys(SHERPA_ONNX_MODEL_CATALOG) as SherpaOnnxModelId[];
+
+export const LOCAL_STT_MODEL_IDS = ALL_MODEL_IDS.filter(
+  (id): id is LocalSttModelId => SHERPA_ONNX_MODEL_CATALOG[id].kind !== "tts"
+);
+
+export const LOCAL_TTS_MODEL_IDS = ALL_MODEL_IDS.filter(
+  (id): id is LocalTtsModelId => SHERPA_ONNX_MODEL_CATALOG[id].kind === "tts"
+);
+
+function resolveDefaultModelId(role: "stt"): LocalSttModelId;
+function resolveDefaultModelId(role: "tts"): LocalTtsModelId;
+function resolveDefaultModelId(role: DefaultModelRole): SherpaOnnxModelId {
+  const match = ALL_MODEL_IDS.find((id) => {
+    const entry: SherpaOnnxCatalogEntry = SHERPA_ONNX_MODEL_CATALOG[id];
+    return entry.defaultFor === role;
+  });
+  if (!match) {
+    throw new Error(`No default model configured for role '${role}'`);
+  }
+  return match;
+}
+
+export const DEFAULT_LOCAL_STT_MODEL = resolveDefaultModelId("stt");
+export const DEFAULT_LOCAL_TTS_MODEL = resolveDefaultModelId("tts");
+
+function buildAliasMap<T extends string>(modelIds: readonly T[]): Record<string, T> {
+  const aliasMap: Record<string, T> = {};
+  for (const modelId of modelIds) {
+    const aliases = SHERPA_ONNX_MODEL_CATALOG[modelId as SherpaOnnxModelId].aliases ?? [];
+    for (const alias of aliases) {
+      aliasMap[alias.trim().toLowerCase()] = modelId;
+    }
+  }
+  return aliasMap;
+}
+
+function createAliasedModelIdSchema<T extends string>(params: {
+  modelIds: readonly T[];
+  aliases: Record<string, T>;
+}): z.ZodType<T, z.ZodTypeDef, string> {
+  const validIds = new Set(params.modelIds);
+  return z
+    .string()
+    .trim()
+    .toLowerCase()
+    .refine(
+      (value): value is T =>
+        validIds.has(value as T) || Object.prototype.hasOwnProperty.call(params.aliases, value),
+      {
+        message: "Invalid model id",
+      }
+    )
+    .transform((value) => params.aliases[value] ?? (value as T));
+}
+
+const STT_MODEL_ALIASES = buildAliasMap(LOCAL_STT_MODEL_IDS);
+const TTS_MODEL_ALIASES = buildAliasMap(LOCAL_TTS_MODEL_IDS);
+
+export const LocalSttModelIdSchema = createAliasedModelIdSchema({
+  modelIds: LOCAL_STT_MODEL_IDS,
+  aliases: STT_MODEL_ALIASES,
+});
+
+export const LocalTtsModelIdSchema = createAliasedModelIdSchema({
+  modelIds: LOCAL_TTS_MODEL_IDS,
+  aliases: TTS_MODEL_ALIASES,
+});
+
+export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & {
+  id: SherpaOnnxModelId;
 };
 
 export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] {
-  return Object.values(SHERPA_ONNX_MODEL_CATALOG);
+  return ALL_MODEL_IDS.map((id) => ({
+    id,
+    ...SHERPA_ONNX_MODEL_CATALOG[id],
+  }));
 }
 
 export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec {
   const spec = SHERPA_ONNX_MODEL_CATALOG[id];
   if (!spec) {
-    throw new Error(`Unknown sherpa-onnx model id: ${id}`);
+    throw new Error(`Unknown local speech model id: ${id}`);
   }
-  return spec;
+  return {
+    id,
+    ...spec,
+  };
 }

packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts

@@ -96,7 +96,7 @@ async function isNonEmptyFile(filePath: string): Promise<boolean> {
 export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOptions): Promise<string> {
   const logger = options.logger.child({
     module: "speech",
-    provider: "sherpa-onnx",
+    provider: "local",
     component: "model-downloader",
     modelId: options.modelId,
   });
@@ -109,8 +109,8 @@ export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOption
 
   if (!options.autoDownload) {
     throw new Error(
-      `Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` +
-        `Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.`
+      `Missing local speech model files for ${options.modelId} in ${modelDir}. ` +
+        `Set PASEO_LOCAL_AUTO_DOWNLOAD=1 to auto-download.`
     );
   }
 

packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts

@@ -34,7 +34,7 @@ export class SherpaOfflineRecognizerEngine {
   private readonly logger: pino.Logger;
 
   constructor(config: SherpaOfflineRecognizerConfig, logger: pino.Logger) {
-    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "offline-recognizer" });
+    this.logger = logger.child({ module: "speech", provider: "local", component: "offline-recognizer" });
 
     assertFileExists(config.model.encoder, "offline encoder");
     assertFileExists(config.model.decoder, "offline decoder");

packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts

@@ -46,7 +46,7 @@ export class SherpaOnlineRecognizerEngine {
   private readonly logger: pino.Logger;
 
   constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) {
-    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "online-recognizer" });
+    this.logger = logger.child({ module: "speech", provider: "local", component: "online-recognizer" });
 
     const { model } = config;
     if (model.kind === "transducer") {

packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts

@@ -12,6 +12,7 @@ export function loadSherpaOnnx(): SherpaOnnxModule {
   if (cached) {
     return cached;
   }
+
   const require = createRequire(import.meta.url);
   cached = require("sherpa-onnx") as SherpaOnnxModule;
   return cached;

packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts

@@ -14,13 +14,16 @@ function platformArch(): string {
   return `${platform}-${process.arch}`;
 }
 
-function prependLibraryPath(envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH", dir: string): void {
+function prependLibraryPath(
+  envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH" | "PATH",
+  dir: string
+): void {
   const current = process.env[envKey] ?? "";
-  const parts = current.split(":").filter(Boolean);
+  const parts = current.split(path.delimiter).filter(Boolean);
   if (parts.includes(dir)) {
     return;
   }
-  process.env[envKey] = [dir, ...parts].join(":");
+  process.env[envKey] = [dir, ...parts].join(path.delimiter);
 }
 
 export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
@@ -31,8 +34,7 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
   const require = createRequire(import.meta.url);
 
   // sherpa-onnx-node depends on a platform-specific package (e.g. sherpa-onnx-darwin-arm64)
-  // that contains the native addon + its shared libraries. On macOS/Linux we need to ensure
-  // the appropriate library path env var includes that directory before requiring the addon.
+  // that contains the native addon and shared libraries. Ensure the OS loader path includes it.
   const arch = platformArch();
   const pkgName = `sherpa-onnx-${arch}`;
 
@@ -43,6 +45,8 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
       prependLibraryPath("DYLD_LIBRARY_PATH", pkgDir);
     } else if (process.platform === "linux") {
       prependLibraryPath("LD_LIBRARY_PATH", pkgDir);
+    } else if (process.platform === "win32") {
+      prependLibraryPath("PATH", pkgDir);
     }
   } catch {
     // Best effort - if the platform package isn't present, require() below will throw a useful error.

packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts

@@ -25,7 +25,7 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider {
   constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) {
     this.engine = config.engine;
     this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
-    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" });
+    this.logger = logger.child({ module: "speech", provider: "local", component: "parakeet-stt" });
   }
 
   public createSession(params: {

packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts

@@ -28,7 +28,7 @@ export class SherpaOnnxSTT implements SpeechToTextProvider {
     this.engine = config.engine;
     this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
     this.tailPaddingMs = config.tailPaddingMs ?? 500;
-    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" });
+    this.logger = logger.child({ module: "speech", provider: "local", component: "stt" });
   }
 
   public createSession(params: {

packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts

@@ -33,7 +33,7 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
     if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
       throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
     }
-    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "tts" });
+    this.logger = logger.child({ module: "speech", provider: "local", component: "tts" });
     this.speakerId = config.speakerId ?? 0;
     this.speed = config.speed ?? 1.0;
 

packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "vitest";
 import { mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
+import { randomUUID } from "node:crypto";
 import pino from "pino";
 
 import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
@@ -96,7 +97,7 @@ describe("speech models (download E2E)", () => {
       const set = getModelSet();
 
       const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-"));
-      const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx");
+      const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech");
 
       const modelIds: SherpaOnnxModelId[] =
         set === "parakeet-pocket"
@@ -114,14 +115,26 @@ describe("speech models (download E2E)", () => {
         paseoHomeRoot,
         dictationFinalTimeoutMs: 8000,
         speech: {
-          dictationSttProvider: "local",
-          voiceSttProvider: "local",
-          voiceTtsProvider: "local",
-          sherpaOnnx: {
+          providers: {
+            dictationStt: { provider: "local", explicit: true },
+            voiceStt: { provider: "local", explicit: true },
+            voiceTts: { provider: "local", explicit: true },
+          },
+          local: {
             modelsDir,
             autoDownload: false,
-            stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" },
-            tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" },
+            models: {
+              dictationStt:
+                set === "parakeet-pocket"
+                  ? "parakeet-tdt-0.6b-v3-int8"
+                  : "zipformer-bilingual-zh-en-2023-02-20",
+              voiceStt:
+                set === "parakeet-pocket"
+                  ? "parakeet-tdt-0.6b-v3-int8"
+                  : "zipformer-bilingual-zh-en-2023-02-20",
+              voiceTts:
+                set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
+            },
           },
         },
       });
@@ -178,7 +191,7 @@ describe("speech models (download E2E)", () => {
           };
         });
 
-        await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`);
+        await ctx.client.setVoiceMode(true, randomUUID());
         for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
           const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
           const isLast = offset + chunkBytes >= pcm16.length;
@@ -188,7 +201,7 @@ describe("speech models (download E2E)", () => {
         if (voiceText.length > 0) {
           expect(voiceText).toContain("voice note");
         }
-        await ctx.client.setVoiceConversation(false);
+        await ctx.client.setVoiceMode(false);
 
         // Streaming TTS: generate locally from downloaded model and validate chunking.
         const ttsText = "This is a voice note.";

packages/server/src/server/speech/providers/openai/config.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, test } from "vitest";
+
+import { PersistedConfigSchema } from "../../../persisted-config.js";
+import { resolveOpenAiSpeechConfig } from "./config.js";
+
+describe("resolveOpenAiSpeechConfig", () => {
+  test("treats empty OPENAI_API_KEY as unset", () => {
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {
+      OPENAI_API_KEY: "",
+    } as NodeJS.ProcessEnv;
+
+    const resolved = resolveOpenAiSpeechConfig({
+      env,
+      persisted,
+      providers: {
+        dictationStt: { provider: "local", explicit: false },
+        voiceStt: { provider: "local", explicit: false },
+        voiceTts: { provider: "local", explicit: false },
+      },
+    });
+
+    expect(resolved).toBeUndefined();
+  });
+
+  test("uses trimmed OPENAI_API_KEY when configured", () => {
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {
+      OPENAI_API_KEY: "  sk-test  ",
+    } as NodeJS.ProcessEnv;
+
+    const resolved = resolveOpenAiSpeechConfig({
+      env,
+      persisted,
+      providers: {
+        dictationStt: { provider: "openai", explicit: true },
+        voiceStt: { provider: "openai", explicit: true },
+        voiceTts: { provider: "openai", explicit: true },
+      },
+    });
+
+    expect(resolved?.apiKey).toBe("sk-test");
+    expect(resolved?.stt?.apiKey).toBe("sk-test");
+    expect(resolved?.tts?.apiKey).toBe("sk-test");
+  });
+});

packages/server/src/server/speech/providers/openai/config.ts

@@ -0,0 +1,126 @@
+import { z } from "zod";
+
+import type { PersistedConfig } from "../../../persisted-config.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import type { STTConfig } from "./stt.js";
+import type { TTSConfig } from "./tts.js";
+
+export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
+export const DEFAULT_OPENAI_TTS_MODEL = "tts-1";
+
+export type OpenAiSpeechProviderConfig = {
+  apiKey?: string;
+  stt?: Partial<STTConfig> & { apiKey?: string };
+  tts?: Partial<TTSConfig> & { apiKey?: string };
+  realtimeTranscriptionModel?: string;
+};
+
+const OpenAiTtsVoiceSchema = z.enum([
+  "alloy",
+  "echo",
+  "fable",
+  "onyx",
+  "nova",
+  "shimmer",
+]);
+
+const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
+
+const NumberLikeSchema = z.union([
+  z.number(),
+  z.string().trim().min(1),
+]);
+
+const OptionalFiniteNumberSchema = NumberLikeSchema
+  .pipe(z.coerce.number().finite())
+  .optional();
+
+const OptionalTrimmedStringSchema = z
+  .string()
+  .trim()
+  .optional()
+  .transform((value) => (value && value.length > 0 ? value : undefined));
+
+const OpenAiSpeechResolutionSchema = z.object({
+  apiKey: OptionalTrimmedStringSchema,
+  sttConfidenceThreshold: OptionalFiniteNumberSchema,
+  sttModel: OptionalTrimmedStringSchema,
+  ttsVoice: z
+    .string()
+    .trim()
+    .toLowerCase()
+    .pipe(OpenAiTtsVoiceSchema)
+    .default("alloy"),
+  ttsModel: z
+    .string()
+    .trim()
+    .toLowerCase()
+    .pipe(OpenAiTtsModelSchema)
+    .default(DEFAULT_OPENAI_TTS_MODEL),
+  realtimeTranscriptionModel: OptionalTrimmedStringSchema.default(
+    DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL
+  ),
+});
+
+export function resolveOpenAiSpeechConfig(params: {
+  env: NodeJS.ProcessEnv;
+  persisted: PersistedConfig;
+  providers: RequestedSpeechProviders;
+}): OpenAiSpeechProviderConfig | undefined {
+  const parsed = OpenAiSpeechResolutionSchema.parse({
+    apiKey: params.env.OPENAI_API_KEY ?? params.persisted.providers?.openai?.apiKey,
+    sttConfidenceThreshold:
+      params.env.STT_CONFIDENCE_THRESHOLD ??
+      params.persisted.features?.dictation?.stt?.confidenceThreshold,
+    sttModel:
+      params.env.STT_MODEL ??
+      (params.providers.voiceStt.provider === "openai"
+        ? params.persisted.features?.voiceMode?.stt?.model
+        : undefined) ??
+      (params.providers.dictationStt.provider === "openai"
+        ? params.persisted.features?.dictation?.stt?.model
+        : undefined),
+    ttsVoice:
+      params.env.TTS_VOICE ??
+      (params.providers.voiceTts.provider === "openai"
+        ? params.persisted.features?.voiceMode?.tts?.voice
+        : undefined) ??
+      "alloy",
+    ttsModel:
+      params.env.TTS_MODEL ??
+      (params.providers.voiceTts.provider === "openai"
+        ? params.persisted.features?.voiceMode?.tts?.model
+        : undefined) ??
+      DEFAULT_OPENAI_TTS_MODEL,
+    realtimeTranscriptionModel:
+      params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ??
+      (params.providers.dictationStt.provider === "openai"
+        ? params.persisted.features?.dictation?.stt?.model
+        : undefined) ??
+      DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+  });
+
+  if (!parsed.apiKey) {
+    return undefined;
+  }
+
+  return {
+    apiKey: parsed.apiKey,
+    stt: {
+      apiKey: parsed.apiKey,
+      ...(parsed.sttConfidenceThreshold !== undefined
+        ? { confidenceThreshold: parsed.sttConfidenceThreshold }
+        : {}),
+      ...(parsed.sttModel
+        ? { model: parsed.sttModel }
+        : {}),
+    },
+    tts: {
+      apiKey: parsed.apiKey,
+      voice: parsed.ttsVoice,
+      model: parsed.ttsModel,
+      responseFormat: "pcm",
+    },
+    realtimeTranscriptionModel: parsed.realtimeTranscriptionModel,
+  };
+}

packages/server/src/server/speech/providers/openai/runtime.ts

@@ -0,0 +1,171 @@
+import type { Logger } from "pino";
+
+import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import {
+  DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+  DEFAULT_OPENAI_TTS_MODEL,
+  type OpenAiSpeechProviderConfig,
+} from "./config.js";
+import { OpenAIRealtimeTranscriptionSession } from "./realtime-transcription-session.js";
+import { OpenAISTT } from "./stt.js";
+import { OpenAITTS } from "./tts.js";
+
+type OpenAiCredentialState = {
+  openaiSttApiKey: string | undefined;
+  openaiTtsApiKey: string | undefined;
+  openaiDictationApiKey: string | undefined;
+};
+
+export type OpenAiSpeechAvailability = {
+  stt: boolean;
+  tts: boolean;
+  dictationStt: boolean;
+};
+
+export type SpeechServices = {
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+  dictationSttService: SpeechToTextProvider | null;
+};
+
+function resolveOpenAiCredentials(
+  openaiConfig: OpenAiSpeechProviderConfig | undefined
+): OpenAiCredentialState {
+  const openaiApiKey = openaiConfig?.apiKey;
+  return {
+    openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey,
+    openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey,
+    openaiDictationApiKey: openaiApiKey,
+  };
+}
+
+export function getOpenAiSpeechAvailability(
+  openaiConfig: OpenAiSpeechProviderConfig | undefined
+): OpenAiSpeechAvailability {
+  const credentials = resolveOpenAiCredentials(openaiConfig);
+  return {
+    stt: Boolean(credentials.openaiSttApiKey),
+    tts: Boolean(credentials.openaiTtsApiKey),
+    dictationStt: Boolean(credentials.openaiDictationApiKey),
+  };
+}
+
+export function validateOpenAiCredentialRequirements(params: {
+  providers: RequestedSpeechProviders;
+  openaiConfig: OpenAiSpeechProviderConfig | undefined;
+  logger: Logger;
+}): void {
+  const { providers, logger, openaiConfig } = params;
+  const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
+
+  const missingOpenAiCredentialsFor: string[] = [];
+  if (providers.voiceStt.provider === "openai" && !openAiCredentials.openaiSttApiKey) {
+    missingOpenAiCredentialsFor.push("voice.stt");
+  }
+  if (providers.voiceTts.provider === "openai" && !openAiCredentials.openaiTtsApiKey) {
+    missingOpenAiCredentialsFor.push("voice.tts");
+  }
+  if (
+    providers.dictationStt.provider === "openai" &&
+    !openAiCredentials.openaiDictationApiKey
+  ) {
+    missingOpenAiCredentialsFor.push("dictation.stt");
+  }
+
+  if (missingOpenAiCredentialsFor.length > 0) {
+    logger.error(
+      {
+        requestedProviders: {
+          dictationStt: providers.dictationStt.provider,
+          voiceStt: providers.voiceStt.provider,
+          voiceTts: providers.voiceTts.provider,
+        },
+        missingOpenAiCredentialsFor,
+      },
+      "Invalid speech configuration: OpenAI provider selected but credentials are missing"
+    );
+    throw new Error(
+      `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
+    );
+  }
+}
+
+export function initializeOpenAiSpeechServices(params: {
+  providers: RequestedSpeechProviders;
+  openaiConfig: OpenAiSpeechProviderConfig | undefined;
+  existing: SpeechServices;
+  logger: Logger;
+}): SpeechServices {
+  const { providers, openaiConfig, existing, logger } = params;
+  const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
+
+  let sttService = existing.sttService;
+  let ttsService = existing.ttsService;
+  let dictationSttService = existing.dictationSttService;
+
+  const needsOpenAiStt = !sttService && providers.voiceStt.provider === "openai";
+  const needsOpenAiTts = !ttsService && providers.voiceTts.provider === "openai";
+  const needsOpenAiDictation =
+    !dictationSttService && providers.dictationStt.provider === "openai";
+
+  if (
+    (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
+    (openAiCredentials.openaiSttApiKey ||
+      openAiCredentials.openaiTtsApiKey ||
+      openAiCredentials.openaiDictationApiKey)
+  ) {
+    logger.info("OpenAI speech provider initialized");
+
+    if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) {
+      const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {};
+      sttService = new OpenAISTT(
+        {
+          apiKey: openAiCredentials.openaiSttApiKey,
+          ...sttConfig,
+        },
+        logger
+      );
+    }
+
+    if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) {
+      const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {};
+      ttsService = new OpenAITTS(
+        {
+          apiKey: openAiCredentials.openaiTtsApiKey,
+          voice: "alloy",
+          model: DEFAULT_OPENAI_TTS_MODEL,
+          responseFormat: "pcm",
+          ...ttsConfig,
+        },
+        logger
+      );
+    }
+
+    const dictationApiKey = openAiCredentials.openaiDictationApiKey;
+    if (needsOpenAiDictation && dictationApiKey) {
+      dictationSttService = {
+        id: "openai",
+        createSession: ({ logger: sessionLogger, language, prompt }) =>
+          new OpenAIRealtimeTranscriptionSession({
+            apiKey: dictationApiKey,
+            logger: sessionLogger,
+            transcriptionModel:
+              openaiConfig?.realtimeTranscriptionModel ??
+              DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+            ...(language ? { language } : {}),
+            ...(prompt ? { prompt } : {}),
+            turnDetection: null,
+          }),
+      };
+    }
+  } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
+    logger.warn("OpenAI speech providers are configured but credentials are missing");
+  }
+
+  return {
+    sttService,
+    ttsService,
+    dictationSttService,
+  };
+}

packages/server/src/server/speech/speech-config-resolver.test.ts

@@ -0,0 +1,127 @@
+import path from "node:path";
+
+import { describe, expect, test } from "vitest";
+
+import { PersistedConfigSchema } from "../persisted-config.js";
+import { resolveSpeechConfig } from "./speech-config-resolver.js";
+
+describe("resolveSpeechConfig", () => {
+  test("resolves local-first defaults without env overrides", () => {
+    const paseoHome = "/tmp/paseo-home";
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {} as NodeJS.ProcessEnv;
+
+    const result = resolveSpeechConfig({
+      paseoHome,
+      env,
+      persisted,
+    });
+
+    expect(result.openai).toBeUndefined();
+    expect(result.speech.providers.dictationStt).toEqual({
+      provider: "local",
+      explicit: false,
+    });
+    expect(result.speech.providers.voiceStt).toEqual({
+      provider: "local",
+      explicit: false,
+    });
+    expect(result.speech.providers.voiceTts).toEqual({
+      provider: "local",
+      explicit: false,
+    });
+    expect(result.speech.local).toEqual({
+      modelsDir: path.join(paseoHome, "models", "local-speech"),
+      autoDownload: true,
+      models: {
+        dictationStt: "parakeet-tdt-0.6b-v3-int8",
+        voiceStt: "parakeet-tdt-0.6b-v3-int8",
+        voiceTts: "pocket-tts-onnx-int8",
+      },
+    });
+    expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
+  });
+
+  test("resolves feature-scoped local model env vars", () => {
+    const persisted = PersistedConfigSchema.parse({
+      features: {
+        voiceMode: {
+          stt: { provider: "openai", model: "gpt-4o-transcribe" },
+        },
+      },
+      providers: {
+        openai: { apiKey: "persisted-key" },
+      },
+    });
+    const env = {
+      PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer",
+      PASEO_VOICE_LOCAL_STT_MODEL: "parakeet",
+      PASEO_VOICE_LOCAL_TTS_MODEL: "kitten",
+      PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5",
+      PASEO_VOICE_LOCAL_TTS_SPEED: "1.35",
+      PASEO_LOCAL_MODELS_DIR: "/tmp/models",
+      PASEO_LOCAL_AUTO_DOWNLOAD: "0",
+      OPENAI_API_KEY: "env-key",
+      PASEO_VOICE_STT_PROVIDER: "openai",
+      PASEO_DICTATION_STT_PROVIDER: "local",
+      PASEO_VOICE_TTS_PROVIDER: "local",
+    } as NodeJS.ProcessEnv;
+
+    const result = resolveSpeechConfig({
+      paseoHome: "/tmp/paseo-home",
+      env,
+      persisted,
+    });
+
+    expect(result.speech.local).toEqual({
+      modelsDir: "/tmp/models",
+      autoDownload: false,
+      models: {
+        dictationStt: "zipformer-bilingual-zh-en-2023-02-20",
+        voiceStt: "parakeet-tdt-0.6b-v3-int8",
+        voiceTts: "kitten-nano-en-v0_1-fp16",
+        voiceTtsSpeakerId: 5,
+        voiceTtsSpeed: 1.35,
+      },
+    });
+    expect(result.speech.providers.dictationStt).toEqual({
+      provider: "local",
+      explicit: true,
+    });
+    expect(result.speech.providers.voiceStt).toEqual({
+      provider: "openai",
+      explicit: true,
+    });
+    expect(result.speech.providers.voiceTts).toEqual({
+      provider: "local",
+      explicit: true,
+    });
+    expect(result.speech.local?.models.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20");
+    expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16");
+    expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5);
+    expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35);
+    expect(result.openai?.apiKey).toBe("env-key");
+    expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe");
+  });
+
+  test("ignores deprecated shared local model env vars", () => {
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {
+      PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20",
+      PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16",
+    } as NodeJS.ProcessEnv;
+
+    const result = resolveSpeechConfig({
+      paseoHome: "/tmp/paseo-home",
+      env,
+      persisted,
+    });
+
+    expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
+  });
+});

packages/server/src/server/speech/speech-config-resolver.ts

@@ -0,0 +1,105 @@
+import { z } from "zod";
+
+import type { PersistedConfig } from "../persisted-config.js";
+import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
+import { resolveLocalSpeechConfig } from "./providers/local/config.js";
+import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js";
+import {
+  SpeechProviderIdSchema,
+  type RequestedSpeechProvider,
+  type RequestedSpeechProviders,
+} from "./speech-types.js";
+
+const OptionalSpeechProviderSchema = z
+  .string()
+  .trim()
+  .toLowerCase()
+  .pipe(SpeechProviderIdSchema)
+  .optional();
+
+const RequestedSpeechProvidersSchema = z.object({
+  dictationStt: OptionalSpeechProviderSchema.default("local"),
+  voiceStt: OptionalSpeechProviderSchema.default("local"),
+  voiceTts: OptionalSpeechProviderSchema.default("local"),
+});
+
+function resolveRequestedSpeechProviders(params: {
+  env: NodeJS.ProcessEnv;
+  persisted: PersistedConfig;
+}): RequestedSpeechProviders {
+  const resolveFeatureProvider = (
+    configuredValue: string | undefined,
+    parsedValue: z.infer<typeof SpeechProviderIdSchema>
+  ): RequestedSpeechProvider => ({
+    provider: parsedValue,
+    explicit: configuredValue !== undefined,
+  });
+
+  const dictationSttProviderFromConfig =
+    params.env.PASEO_DICTATION_STT_PROVIDER ??
+    params.persisted.features?.dictation?.stt?.provider;
+  const voiceSttProviderFromConfig =
+    params.env.PASEO_VOICE_STT_PROVIDER ??
+    params.persisted.features?.voiceMode?.stt?.provider;
+  const voiceTtsProviderFromConfig =
+    params.env.PASEO_VOICE_TTS_PROVIDER ??
+    params.persisted.features?.voiceMode?.tts?.provider;
+
+  const parsed = RequestedSpeechProvidersSchema.parse({
+    dictationStt: dictationSttProviderFromConfig ?? "local",
+    voiceStt: voiceSttProviderFromConfig ?? "local",
+    voiceTts: voiceTtsProviderFromConfig ?? "local",
+  });
+
+  return {
+    dictationStt: resolveFeatureProvider(
+      dictationSttProviderFromConfig,
+      parsed.dictationStt
+    ),
+    voiceStt: resolveFeatureProvider(
+      voiceSttProviderFromConfig,
+      parsed.voiceStt
+    ),
+    voiceTts: resolveFeatureProvider(
+      voiceTtsProviderFromConfig,
+      parsed.voiceTts
+    ),
+  };
+}
+
+export function resolveSpeechConfig(params: {
+  paseoHome: string;
+  env: NodeJS.ProcessEnv;
+  persisted: PersistedConfig;
+}): {
+  openai: PaseoOpenAIConfig | undefined;
+  speech: PaseoSpeechConfig;
+} {
+  const providers = resolveRequestedSpeechProviders({
+    env: params.env,
+    persisted: params.persisted,
+  });
+
+  const local = resolveLocalSpeechConfig({
+    paseoHome: params.paseoHome,
+    env: params.env,
+    persisted: params.persisted,
+    providers,
+  });
+
+  const openai = resolveOpenAiSpeechConfig({
+    env: params.env,
+    persisted: params.persisted,
+    providers,
+  });
+
+  return {
+    openai,
+    speech: {
+      providers,
+      ...(local.local
+        ? { local: local.local }
+        : {}),
+    },
+  };
+}

packages/server/src/server/speech/speech-runtime.ts

@@ -0,0 +1,173 @@
+import type { Logger } from "pino";
+
+import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
+import {
+  getLocalSpeechAvailability,
+  initializeLocalSpeechServices,
+} from "./providers/local/runtime.js";
+import type { LocalSpeechModelId } from "./providers/local/config.js";
+import {
+  getOpenAiSpeechAvailability,
+  initializeOpenAiSpeechServices,
+  validateOpenAiCredentialRequirements,
+} from "./providers/openai/runtime.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
+import type { RequestedSpeechProviders } from "./speech-types.js";
+
+function resolveRequestedSpeechProviders(
+  speechConfig: PaseoSpeechConfig | null
+): RequestedSpeechProviders {
+  const fromConfig = speechConfig?.providers;
+  if (fromConfig) {
+    return fromConfig;
+  }
+
+  return {
+    dictationStt: { provider: "local", explicit: false },
+    voiceStt: { provider: "local", explicit: false },
+    voiceTts: { provider: "local", explicit: false },
+  };
+}
+
+export type InitializedSpeechRuntime = {
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+  dictationSttService: SpeechToTextProvider | null;
+  cleanup: () => void;
+  localModelConfig: {
+    modelsDir: string;
+    defaultModelIds: LocalSpeechModelId[];
+  } | null;
+};
+
+export async function initializeSpeechRuntime(params: {
+  logger: Logger;
+  openaiConfig?: PaseoOpenAIConfig;
+  speechConfig?: PaseoSpeechConfig;
+}): Promise<InitializedSpeechRuntime> {
+  const logger = params.logger;
+  const speechConfig = params.speechConfig ?? null;
+  const openaiConfig = params.openaiConfig;
+  const providers = resolveRequestedSpeechProviders(speechConfig);
+
+  validateOpenAiCredentialRequirements({
+    providers,
+    openaiConfig,
+    logger,
+  });
+
+  logger.info(
+    {
+      requestedProviders: {
+        dictationStt: providers.dictationStt.provider,
+        voiceStt: providers.voiceStt.provider,
+        voiceTts: providers.voiceTts.provider,
+      },
+      availability: {
+        openai: getOpenAiSpeechAvailability(openaiConfig),
+        local: getLocalSpeechAvailability(speechConfig),
+      },
+    },
+    "Speech provider reconciliation started"
+  );
+
+  const localSpeech = await initializeLocalSpeechServices({
+    providers,
+    speechConfig,
+    logger,
+  });
+
+  const openAiSpeech = initializeOpenAiSpeechServices({
+    providers,
+    openaiConfig,
+    existing: {
+      sttService: localSpeech.sttService,
+      ttsService: localSpeech.ttsService,
+      dictationSttService: localSpeech.dictationSttService,
+    },
+    logger,
+  });
+
+  const effectiveProviders = {
+    dictationStt: openAiSpeech.dictationSttService?.id ?? "unavailable",
+    voiceStt: openAiSpeech.sttService?.id ?? "unavailable",
+    voiceTts:
+      !openAiSpeech.ttsService
+        ? "unavailable"
+        : openAiSpeech.ttsService === localSpeech.localVoiceTtsProvider
+          ? "local"
+          : "openai",
+  };
+  const unavailableFeatures = [
+    !openAiSpeech.dictationSttService ? "dictation.stt" : null,
+    !openAiSpeech.sttService ? "voice.stt" : null,
+    !openAiSpeech.ttsService ? "voice.tts" : null,
+  ].filter((feature): feature is string => feature !== null);
+  const explicitlyConfiguredUnavailableFeatures = unavailableFeatures.filter((feature) => {
+    if (feature === "dictation.stt") {
+      return providers.dictationStt.explicit;
+    }
+    if (feature === "voice.stt") {
+      return providers.voiceStt.explicit;
+    }
+    return providers.voiceTts.explicit;
+  });
+
+  if (explicitlyConfiguredUnavailableFeatures.length > 0) {
+    logger.error(
+      {
+        requestedProviders: {
+          dictationStt: providers.dictationStt.provider,
+          voiceStt: providers.voiceStt.provider,
+          voiceTts: providers.voiceTts.provider,
+        },
+        explicitProviders: {
+          dictationStt: providers.dictationStt.explicit,
+          voiceStt: providers.voiceStt.explicit,
+          voiceTts: providers.voiceTts.explicit,
+        },
+        effectiveProviders,
+        unavailableFeatures: explicitlyConfiguredUnavailableFeatures,
+      },
+      "Speech provider reconciliation failed: configured features are unavailable"
+    );
+    throw new Error(
+      `Configured speech features unavailable: ${explicitlyConfiguredUnavailableFeatures.join(", ")}`
+    );
+  }
+
+  if (unavailableFeatures.length > 0) {
+    logger.warn(
+      {
+        requestedProviders: {
+          dictationStt: providers.dictationStt.provider,
+          voiceStt: providers.voiceStt.provider,
+          voiceTts: providers.voiceTts.provider,
+        },
+        explicitProviders: {
+          dictationStt: providers.dictationStt.explicit,
+          voiceStt: providers.voiceStt.explicit,
+          voiceTts: providers.voiceTts.explicit,
+        },
+        effectiveProviders,
+        unavailableFeatures,
+      },
+      "Speech provider reconciliation completed with unavailable default features"
+    );
+  } else {
+    logger.info(
+      {
+        effectiveProviders,
+      },
+      "Speech provider reconciliation completed"
+    );
+  }
+
+  return {
+    sttService: openAiSpeech.sttService,
+    ttsService: openAiSpeech.ttsService,
+    dictationSttService: openAiSpeech.dictationSttService,
+    cleanup: localSpeech.cleanup,
+    localModelConfig: localSpeech.localModelConfig,
+  };
+}

packages/server/src/server/speech/speech-types.ts

@@ -0,0 +1,16 @@
+import { z } from "zod";
+
+export const SpeechProviderIdSchema = z.enum(["openai", "local"]);
+export type SpeechProviderId = z.infer<typeof SpeechProviderIdSchema>;
+
+export const RequestedSpeechProviderSchema = z.object({
+  provider: SpeechProviderIdSchema,
+  explicit: z.boolean(),
+});
+export type RequestedSpeechProvider = z.infer<typeof RequestedSpeechProviderSchema>;
+
+export type RequestedSpeechProviders = {
+  dictationStt: RequestedSpeechProvider;
+  voiceStt: RequestedSpeechProvider;
+  voiceTts: RequestedSpeechProvider;
+};

packages/server/src/server/test-utils/paseo-daemon.ts

@@ -21,6 +21,9 @@ type TestPaseoDaemonOptions = {
   cleanup?: boolean;
   openai?: PaseoOpenAIConfig;
   speech?: PaseoSpeechConfig;
+  voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"];
+  voiceLlmProviderExplicit?: boolean;
+  voiceLlmModel?: string | null;
   dictationFinalTimeoutMs?: number;
 };
 
@@ -78,7 +81,9 @@ export async function createTestPaseoDaemon(
       appBaseUrl: "https://app.paseo.sh",
       openai: options.openai,
       speech: options.speech,
-      openrouterApiKey: null,
+      voiceLlmProvider: options.voiceLlmProvider ?? null,
+      voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false,
+      voiceLlmModel: options.voiceLlmModel ?? null,
       dictationFinalTimeoutMs: options.dictationFinalTimeoutMs,
       downloadTokenTtlMs: options.downloadTokenTtlMs,
     };

packages/server/src/server/voice-conversation-store.ts

@@ -1,140 +0,0 @@
-import { readFile, writeFile, readdir, unlink, mkdir, stat } from "fs/promises";
-import { join } from "path";
-import type { ModelMessage } from "@ai-sdk/provider-utils";
-import { standardizePrompt } from "ai/internal";
-
-type LoggerLike = {
-  child(bindings: Record<string, unknown>): LoggerLike;
-  info(...args: any[]): void;
-  debug(...args: any[]): void;
-  warn(...args: any[]): void;
-  error(...args: any[]): void;
-};
-
-function getLogger(logger: LoggerLike): LoggerLike {
-  return logger.child({ module: "voice-conversation-store" });
-}
-
-export interface VoiceConversationMetadata {
-  id: string;
-  lastUpdated: Date;
-  messageCount: number;
-}
-
-interface VoiceConversationData {
-  voiceConversationId: string;
-  lastUpdated: string;
-  messageCount: number;
-  messages: ModelMessage[];
-}
-
-export class VoiceConversationStore {
-  private readonly baseDir: string;
-
-  constructor(baseDir: string) {
-    this.baseDir = baseDir;
-  }
-
-  private async ensureBaseDir(): Promise<void> {
-    await mkdir(this.baseDir, { recursive: true });
-  }
-
-  public async save(
-    logger: LoggerLike,
-    voiceConversationId: string,
-    messages: ModelMessage[]
-  ): Promise<void> {
-    const log = getLogger(logger);
-    await this.ensureBaseDir();
-
-    const filepath = join(this.baseDir, `${voiceConversationId}.json`);
-    const data: VoiceConversationData = {
-      voiceConversationId,
-      lastUpdated: new Date().toISOString(),
-      messageCount: messages.length,
-      messages,
-    };
-
-    await writeFile(filepath, JSON.stringify(data, null, 2), "utf-8");
-    log.debug({ voiceConversationId, messageCount: messages.length }, "Saved voice conversation");
-  }
-
-  /**
-   * Load voice conversation from disk.
-   * Returns null when missing or invalid (best-effort).
-   */
-  public async load(
-    logger: LoggerLike,
-    voiceConversationId: string
-  ): Promise<ModelMessage[] | null> {
-    const log = getLogger(logger);
-    const filepath = join(this.baseDir, `${voiceConversationId}.json`);
-
-    try {
-      await stat(filepath);
-    } catch {
-      log.debug({ voiceConversationId }, "Voice conversation not found");
-      return null;
-    }
-
-    try {
-      const fileContent = await readFile(filepath, "utf-8");
-      const data: VoiceConversationData = JSON.parse(fileContent);
-
-      const result = await standardizePrompt({ prompt: data.messages });
-
-      log.debug(
-        { voiceConversationId, messageCount: data.messageCount },
-        "Loaded voice conversation"
-      );
-
-      return result.messages as ModelMessage[];
-    } catch (error) {
-      log.warn({ err: error, voiceConversationId }, "Failed to load voice conversation");
-      return null;
-    }
-  }
-
-  public async list(logger: LoggerLike): Promise<VoiceConversationMetadata[]> {
-    const log = getLogger(logger);
-    try {
-      await this.ensureBaseDir();
-
-      const files = await readdir(this.baseDir);
-      const jsonFiles = files.filter((f) => f.endsWith(".json"));
-      const conversations: VoiceConversationMetadata[] = [];
-
-      for (const file of jsonFiles) {
-        try {
-          const filepath = join(this.baseDir, file);
-          const fileContent = await readFile(filepath, "utf-8");
-          const data: VoiceConversationData = JSON.parse(fileContent);
-
-          conversations.push({
-            id: data.voiceConversationId,
-            lastUpdated: new Date(data.lastUpdated),
-            messageCount: data.messageCount,
-          });
-        } catch (error) {
-          log.warn({ err: error, file }, "Failed to read voice conversation file");
-        }
-      }
-
-      conversations.sort(
-        (a, b) => b.lastUpdated.getTime() - a.lastUpdated.getTime()
-      );
-      return conversations;
-    } catch (error) {
-      log.warn({ err: error }, "Failed to list voice conversations");
-      return [];
-    }
-  }
-
-  public async delete(logger: LoggerLike, voiceConversationId: string): Promise<void> {
-    const log = getLogger(logger);
-    const filepath = join(this.baseDir, `${voiceConversationId}.json`);
-    await unlink(filepath);
-    log.debug({ voiceConversationId }, "Deleted voice conversation");
-  }
-}
-

packages/server/src/server/voice-conversations.e2e.test.ts

@@ -1,147 +0,0 @@
-import { describe, test, expect } from "vitest";
-import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
-import { v4 as uuidv4 } from "uuid";
-
-import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js";
-import { DaemonClient } from "./test-utils/daemon-client.js";
-
-async function waitForFile(filepath: string, timeoutMs = 5000): Promise<void> {
-  const start = Date.now();
-  // eslint-disable-next-line no-constant-condition
-  while (true) {
-    if (existsSync(filepath)) {
-      return;
-    }
-    if (Date.now() - start > timeoutMs) {
-      throw new Error(`Timed out waiting for file: ${filepath}`);
-    }
-    await new Promise((r) => setTimeout(r, 50));
-  }
-}
-
-async function waitForJsonFile<T>(
-  filepath: string,
-  timeoutMs = 5000
-): Promise<T> {
-  const start = Date.now();
-  // eslint-disable-next-line no-constant-condition
-  while (true) {
-    if (existsSync(filepath)) {
-      try {
-        const raw = readFileSync(filepath, "utf8");
-        if (raw.trim().length > 0) {
-          return JSON.parse(raw) as T;
-        }
-      } catch {
-        // File may exist but still be mid-write; retry.
-      }
-    }
-    if (Date.now() - start > timeoutMs) {
-      throw new Error(`Timed out waiting for valid JSON: ${filepath}`);
-    }
-    await new Promise((r) => setTimeout(r, 50));
-  }
-}
-
-describe("voice conversations - daemon E2E", () => {
-  test(
-    "two concurrent clients persist independently under paseoHome/voice-conversations",
-    async () => {
-      const daemon = await createTestPaseoDaemon();
-      const url = `ws://127.0.0.1:${daemon.port}/ws`;
-
-      const clientA = new DaemonClient({ url });
-      const clientB = new DaemonClient({ url });
-      await clientA.connect();
-      await clientB.connect();
-
-      try {
-        const voiceConversationIdA = uuidv4();
-        const voiceConversationIdB = uuidv4();
-
-        await clientA.setVoiceConversation(true, voiceConversationIdA);
-        await clientB.setVoiceConversation(true, voiceConversationIdB);
-
-        // Minimal traffic to cause a persist without requiring external APIs.
-        await clientA.setVoiceConversation(false);
-        await clientB.setVoiceConversation(false);
-
-        const fileA = join(
-          daemon.paseoHome,
-          "voice-conversations",
-          `${voiceConversationIdA}.json`
-        );
-        const fileB = join(
-          daemon.paseoHome,
-          "voice-conversations",
-          `${voiceConversationIdB}.json`
-        );
-
-        await waitForFile(fileA);
-        await waitForFile(fileB);
-
-        const dataA = await waitForJsonFile<{
-          voiceConversationId: string;
-          messageCount: number;
-          messages: unknown[];
-        }>(fileA);
-        const dataB = await waitForJsonFile<{
-          voiceConversationId: string;
-          messageCount: number;
-          messages: unknown[];
-        }>(fileB);
-
-        expect(dataA.voiceConversationId).toBe(voiceConversationIdA);
-        expect(dataB.voiceConversationId).toBe(voiceConversationIdB);
-        expect(dataA.messageCount).toBe(0);
-        expect(dataB.messageCount).toBe(0);
-        expect(Array.isArray(dataA.messages)).toBe(true);
-        expect(Array.isArray(dataB.messages)).toBe(true);
-      } finally {
-        await clientA.close().catch(() => undefined);
-        await clientB.close().catch(() => undefined);
-        await daemon.close();
-      }
-    },
-    30000
-  );
-
-  test(
-    "WS attach ignores URL conversationId param for voice conversation state",
-    async () => {
-      const daemon = await createTestPaseoDaemon();
-      const urlConversationId = `url-${uuidv4()}`;
-      const url = `ws://127.0.0.1:${daemon.port}/ws?conversationId=${encodeURIComponent(
-        urlConversationId
-      )}`;
-
-      const client = new DaemonClient({ url });
-      await client.connect();
-
-      try {
-        const voiceConversationId = `client-${uuidv4()}`;
-        await client.setVoiceConversation(true, voiceConversationId);
-        await client.setVoiceConversation(false);
-
-        const file = join(
-          daemon.paseoHome,
-          "voice-conversations",
-          `${voiceConversationId}.json`
-        );
-        const urlFile = join(
-          daemon.paseoHome,
-          "voice-conversations",
-          `${urlConversationId}.json`
-        );
-
-        await waitForFile(file);
-        expect(existsSync(urlFile)).toBe(false);
-      } finally {
-        await client.close().catch(() => undefined);
-        await daemon.close();
-      }
-    },
-    30000
-  );
-});

packages/server/src/server/voice-local-agent.e2e.test.ts

@@ -0,0 +1,136 @@
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+import { readFile } from "node:fs/promises";
+
+import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
+
+const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
+const shouldRun =
+  process.env.PASEO_VOICE_LOCAL_AGENT_E2E === "1" &&
+  Boolean(openaiApiKey) &&
+  !process.env.CI;
+
+function waitForSignal<T>(
+  timeoutMs: number,
+  setup: (
+    resolve: (value: T) => void,
+    reject: (error: Error) => void
+  ) => () => void
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    let cleanup: (() => void) | null = null;
+    const timeout = setTimeout(() => {
+      cleanup?.();
+      reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`));
+    }, timeoutMs);
+
+    cleanup = setup(
+      (value) => {
+        clearTimeout(timeout);
+        cleanup?.();
+        resolve(value);
+      },
+      (error) => {
+        clearTimeout(timeout);
+        cleanup?.();
+        reject(error);
+      }
+    );
+  });
+}
+
+(shouldRun ? describe : describe.skip)(
+  "voice local-agent e2e",
+  () => {
+    let ctx: DaemonTestContext;
+
+    beforeAll(async () => {
+      ctx = await createDaemonTestContext({
+        agentClients: {},
+        openai: { apiKey: openaiApiKey! },
+        speech: {
+          providers: {
+            dictationStt: { provider: "openai", explicit: true },
+            voiceStt: { provider: "openai", explicit: true },
+            voiceTts: { provider: "openai", explicit: true },
+          },
+        },
+        voiceLlmProvider: "codex",
+        voiceLlmProviderExplicit: true,
+        voiceLlmModel: "gpt-5.2-mini",
+      });
+    }, 120000);
+
+    afterAll(async () => {
+      await ctx.cleanup();
+    }, 60000);
+
+    test(
+      "routes voice turns through local agent speak tool",
+      async () => {
+        await ctx.client.setVoiceMode(true, randomUUID());
+
+        const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
+          const offAudio = ctx.client.on("audio_output", (msg) => {
+            if (msg.type !== "audio_output") return;
+            resolve({ chunkId: msg.payload.id });
+          });
+          const offError = ctx.client.on("activity_log", (msg) => {
+            if (msg.type !== "activity_log") return;
+            if (msg.payload.type !== "error") return;
+            reject(new Error(String(msg.payload.content)));
+          });
+          return () => {
+            offAudio();
+            offError();
+          };
+        });
+
+        const assistantLogPromise = waitForSignal<string>(120000, (resolve, reject) => {
+          const offLog = ctx.client.on("activity_log", (msg) => {
+            if (msg.type !== "activity_log") return;
+            if (msg.payload.type !== "assistant") return;
+            const content = String(msg.payload.content ?? "");
+            if (!content.trim()) return;
+            resolve(content);
+          });
+          const offError = ctx.client.on("activity_log", (msg) => {
+            if (msg.type !== "activity_log") return;
+            if (msg.payload.type !== "error") return;
+            reject(new Error(String(msg.payload.content)));
+          });
+          return () => {
+            offLog();
+            offError();
+          };
+        });
+
+        const fixturePath = path.resolve(
+          process.cwd(),
+          "..",
+          "app",
+          "e2e",
+          "fixtures",
+          "recording.wav"
+        );
+        const wav = await readFile(fixturePath);
+        await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true);
+
+        const [{ chunkId }, assistantText] = await Promise.all([
+          audioPromise,
+          assistantLogPromise,
+        ]);
+
+        expect(chunkId.length).toBeGreaterThan(0);
+        expect(assistantText.trim().length).toBeGreaterThan(0);
+
+        const agents = await ctx.client.fetchAgents();
+        expect(
+          agents.some((agent) => String(agent.labels?.surface ?? "") === "voice")
+        ).toBe(false);
+      },
+      180000
+    );
+  }
+);

packages/server/src/server/voice-mcp-bridge.test.ts

@@ -0,0 +1,93 @@
+import os from "node:os";
+import path from "node:path";
+import { mkdtemp, rm } from "node:fs/promises";
+import { describe, expect, test } from "vitest";
+import { experimental_createMCPClient } from "ai";
+import { z } from "zod";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import pino from "pino";
+
+import { createVoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js";
+
+describe("voice MCP bridge", () => {
+  test("proxies stdio MCP messages through unix socket bridge", async () => {
+    const tmpRoot = await mkdtemp(path.join(os.tmpdir(), "paseo-voice-mcp-bridge-"));
+    const socketPath = path.join(tmpRoot, "voice-mcp.sock");
+    const callerAgentId = "voice-agent-bridge-test";
+
+    const bridge = createVoiceMcpBridgeSocketServer({
+      socketPath,
+      logger: pino({ level: "silent" }),
+      createAgentMcpServerForCaller: async (callerId) => {
+        const server = new McpServer({
+          name: "bridge-test-server",
+          version: "1.0.0",
+        });
+
+        server.registerTool(
+          "echo_caller",
+          {
+            value: z.string().optional(),
+          },
+          async (args) => {
+            return {
+              content: [
+                {
+                  type: "text",
+                  text: JSON.stringify({
+                    callerAgentId: callerId,
+                    value: args.value ?? null,
+                  }),
+                },
+              ],
+              structuredContent: {
+                callerAgentId: callerId,
+                value: args.value ?? null,
+              },
+            };
+          }
+        );
+
+        return server;
+      },
+    });
+
+    await bridge.start();
+
+    const tsxBin = path.resolve(process.cwd(), "../../node_modules/.bin/tsx");
+    const serverIndex = path.resolve(process.cwd(), "src/server/index.ts");
+
+    const transport = new StdioClientTransport({
+      command: process.execPath,
+      args: [
+        tsxBin,
+        serverIndex,
+        "__paseo_voice_mcp_bridge",
+        "--socket",
+        socketPath,
+        "--caller-agent-id",
+        callerAgentId,
+      ],
+    });
+
+    const client = await experimental_createMCPClient({ transport });
+
+    try {
+      const result = await client.callTool({
+        name: "echo_caller",
+        args: { value: "ok" },
+      });
+
+      const payload =
+        ((result as { structuredContent?: { callerAgentId?: string; value?: string | null } })
+          .structuredContent) ?? null;
+
+      expect(payload?.callerAgentId).toBe(callerAgentId);
+    } finally {
+      await client.close();
+      await bridge.stop();
+      await rm(tmpRoot, { recursive: true, force: true });
+    }
+  }, 30_000);
+});

packages/server/src/server/voice-mcp-bridge.ts

@@ -0,0 +1,337 @@
+import net from "node:net";
+import path from "node:path";
+import { mkdir, rm } from "node:fs/promises";
+import type { Logger } from "pino";
+import pino from "pino";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import type { JSONRPCMessage } from "@modelcontextprotocol/sdk/types.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+
+
+type BridgeEnvelope =
+  | { type: "init"; callerAgentId: string }
+  | { type: "mcp"; message: JSONRPCMessage }
+  | { type: "ready" }
+  | { type: "error"; message: string };
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+function parseEnvelope(raw: string): BridgeEnvelope {
+  const parsed = JSON.parse(raw);
+  if (!isRecord(parsed) || typeof parsed.type !== "string") {
+    throw new Error("Invalid bridge envelope");
+  }
+  if (parsed.type === "init") {
+    const callerAgentId = typeof parsed.callerAgentId === "string" ? parsed.callerAgentId.trim() : "";
+    if (!callerAgentId) {
+      throw new Error("Invalid init payload: callerAgentId is required");
+    }
+    return { type: "init", callerAgentId };
+  }
+  if (parsed.type === "mcp") {
+    if (!("message" in parsed)) {
+      throw new Error("Invalid mcp payload: message is required");
+    }
+    return { type: "mcp", message: parsed.message as JSONRPCMessage };
+  }
+  if (parsed.type === "ready") return { type: "ready" };
+  if (parsed.type === "error") {
+    return { type: "error", message: typeof parsed.message === "string" ? parsed.message : "Unknown error" };
+  }
+  throw new Error(`Unknown envelope type: ${parsed.type}`);
+}
+
+function encodeEnvelope(envelope: BridgeEnvelope): string {
+  return `${JSON.stringify(envelope)}\n`;
+}
+
+export type VoiceMcpBridgeSocketServer = {
+  socketPath: string;
+  start: () => Promise<void>;
+  stop: () => Promise<void>;
+};
+
+export function createVoiceMcpBridgeSocketServer(params: {
+  socketPath: string;
+  logger: Logger;
+  createAgentMcpServerForCaller: (callerAgentId: string) => Promise<{ connect: (transport: InMemoryTransport) => Promise<void>; close?: () => Promise<void> }>;
+}): VoiceMcpBridgeSocketServer {
+  const logger = params.logger.child({ module: "voice-mcp-bridge" });
+  const sockets = new Set<net.Socket>();
+
+  const server = net.createServer((socket) => {
+    sockets.add(socket);
+    const connectionLogger = logger.child({ component: "connection" });
+    let readBuffer = "";
+    let initialized = false;
+    let clientTransport: InMemoryTransport | null = null;
+    let mcpServer: { close?: () => Promise<void> } | null = null;
+
+    const send = (payload: BridgeEnvelope) => {
+      socket.write(encodeEnvelope(payload));
+    };
+
+    const fail = (message: string) => {
+      send({ type: "error", message });
+      socket.end();
+    };
+
+    const cleanup = async () => {
+      sockets.delete(socket);
+      const closeTasks: Promise<unknown>[] = [];
+      if (clientTransport) {
+        closeTasks.push(clientTransport.close().catch(() => undefined));
+      }
+      if (mcpServer?.close) {
+        closeTasks.push(mcpServer.close().catch(() => undefined));
+      }
+      await Promise.all(closeTasks);
+    };
+
+    socket.on("data", (chunk) => {
+      readBuffer += chunk.toString("utf8");
+      while (true) {
+        const newlineIndex = readBuffer.indexOf("\n");
+        if (newlineIndex < 0) break;
+        const line = readBuffer.slice(0, newlineIndex).trim();
+        readBuffer = readBuffer.slice(newlineIndex + 1);
+        if (!line) continue;
+
+        let message: BridgeEnvelope;
+        try {
+          message = parseEnvelope(line);
+        } catch (error) {
+          fail(error instanceof Error ? error.message : String(error));
+          return;
+        }
+
+        if (message.type === "init") {
+          if (initialized) {
+            fail("Bridge already initialized");
+            return;
+          }
+          initialized = true;
+          void (async () => {
+            try {
+              const [proxyClient, proxyServer] = InMemoryTransport.createLinkedPair();
+              const serverInstance = await params.createAgentMcpServerForCaller(message.callerAgentId);
+              await serverInstance.connect(proxyServer);
+              await proxyClient.start();
+              proxyClient.onmessage = (jsonrpcMessage) => {
+                send({ type: "mcp", message: jsonrpcMessage });
+              };
+              clientTransport = proxyClient;
+              mcpServer = serverInstance;
+              send({ type: "ready" });
+            } catch (error) {
+              connectionLogger.error({ err: error }, "Failed to initialize voice MCP bridge connection");
+              fail(error instanceof Error ? error.message : String(error));
+            }
+          })();
+          continue;
+        }
+
+        if (message.type === "mcp") {
+          if (!clientTransport) {
+            fail("Bridge is not initialized");
+            return;
+          }
+          void clientTransport.send(message.message).catch((error) => {
+            connectionLogger.error({ err: error }, "Failed to forward MCP message");
+            fail(error instanceof Error ? error.message : String(error));
+          });
+          continue;
+        }
+      }
+    });
+
+    socket.on("error", (error) => {
+      connectionLogger.error({ err: error }, "Voice MCP bridge socket error");
+    });
+    socket.on("close", () => {
+      void cleanup();
+    });
+  });
+
+  return {
+    socketPath: params.socketPath,
+    async start() {
+      await mkdir(path.dirname(params.socketPath), { recursive: true });
+      await rm(params.socketPath, { force: true }).catch(() => undefined);
+      await new Promise<void>((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(params.socketPath, () => {
+          server.off("error", reject);
+          resolve();
+        });
+      });
+      logger.info({ socketPath: params.socketPath }, "Voice MCP bridge socket server listening");
+    },
+    async stop() {
+      for (const socket of sockets) {
+        socket.destroy();
+      }
+      await new Promise<void>((resolve, reject) => {
+        server.close((error) => {
+          if (error) reject(error);
+          else resolve();
+        });
+      });
+      await rm(params.socketPath, { force: true }).catch(() => undefined);
+    },
+  };
+}
+
+function parseBridgeCliArgs(argv: string[]): { socketPath: string; callerAgentId: string } {
+  let socketPath: string | null = null;
+  let callerAgentId: string | null = null;
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--socket") {
+      socketPath = argv[index + 1] ?? null;
+      index += 1;
+      continue;
+    }
+    if (arg === "--caller-agent-id") {
+      callerAgentId = argv[index + 1] ?? null;
+      index += 1;
+      continue;
+    }
+  }
+
+  if (!socketPath?.trim()) {
+    throw new Error("Missing required --socket <path>");
+  }
+  if (!callerAgentId?.trim()) {
+    throw new Error("Missing required --caller-agent-id <id>");
+  }
+
+  return {
+    socketPath: socketPath.trim(),
+    callerAgentId: callerAgentId.trim(),
+  };
+}
+
+export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Promise<void> {
+  const bridgeLogger = logger ?? pino({ level: "error" });
+  const parsed = parseBridgeCliArgs(argv);
+
+  const socket = net.createConnection(parsed.socketPath);
+  const stdioTransport = new StdioServerTransport(process.stdin, process.stdout);
+  let socketBuffer = "";
+  let stdioStarted = false;
+  const pendingMcpMessages: JSONRPCMessage[] = [];
+
+  let resolveReady: (() => void) | null = null;
+  let rejectReady: ((error: Error) => void) | null = null;
+  const readyPromise = new Promise<void>((resolve, reject) => {
+    resolveReady = () => {
+      resolveReady = null;
+      rejectReady = null;
+      resolve();
+    };
+    rejectReady = (error: Error) => {
+      resolveReady = null;
+      rejectReady = null;
+      reject(error);
+    };
+  });
+
+  const failReady = (message: string) => {
+    if (rejectReady) {
+      rejectReady(new Error(message));
+    }
+  };
+
+  const sendEnvelope = (payload: BridgeEnvelope) => {
+    socket.write(encodeEnvelope(payload));
+  };
+
+  socket.on("data", (chunk) => {
+    socketBuffer += chunk.toString("utf8");
+    while (true) {
+      const newlineIndex = socketBuffer.indexOf("\n");
+      if (newlineIndex < 0) break;
+      const line = socketBuffer.slice(0, newlineIndex).trim();
+      socketBuffer = socketBuffer.slice(newlineIndex + 1);
+      if (!line) continue;
+
+      let envelope: BridgeEnvelope;
+      try {
+        envelope = parseEnvelope(line);
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        bridgeLogger.error({ err: error }, "Failed to parse voice MCP bridge envelope");
+        failReady(`Failed to parse voice MCP bridge envelope: ${message}`);
+        socket.destroy();
+        return;
+      }
+      if (envelope.type === "ready") {
+        resolveReady?.();
+        continue;
+      }
+      if (envelope.type === "error") {
+        failReady(`Voice MCP bridge error: ${envelope.message}`);
+        socket.destroy();
+        return;
+      }
+      if (envelope.type === "mcp") {
+        if (!stdioStarted) {
+          pendingMcpMessages.push(envelope.message);
+          continue;
+        }
+        void stdioTransport.send(envelope.message).catch((error) => {
+          bridgeLogger.error({ err: error }, "Failed to forward MCP message to stdio transport");
+          socket.destroy();
+        });
+      }
+    }
+  });
+
+  socket.on("error", (error) => {
+    bridgeLogger.error({ err: error }, "Voice MCP bridge socket client error");
+    failReady(`Voice MCP bridge socket client error: ${error.message}`);
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    socket.once("connect", () => resolve());
+    socket.once("error", reject);
+  });
+
+  sendEnvelope({ type: "init", callerAgentId: parsed.callerAgentId });
+
+  const readyTimeoutPromise = new Promise<void>((_, reject) => {
+    const timeout = setTimeout(() => {
+      reject(new Error("Timed out waiting for voice MCP bridge initialization"));
+    }, 10000);
+    readyPromise.finally(() => clearTimeout(timeout)).catch(() => undefined);
+  });
+  await Promise.race([readyPromise, readyTimeoutPromise]);
+
+  stdioTransport.onmessage = (message) => {
+    sendEnvelope({ type: "mcp", message });
+  };
+  stdioTransport.onerror = (error) => {
+    bridgeLogger.error({ err: error }, "Voice MCP stdio transport error");
+    socket.destroy();
+  };
+  stdioTransport.onclose = () => {
+    socket.end();
+  };
+
+  await stdioTransport.start();
+  stdioStarted = true;
+  for (const message of pendingMcpMessages) {
+    await stdioTransport.send(message);
+  }
+
+  await new Promise<void>((resolve) => {
+    socket.once("close", () => resolve());
+    process.stdin.once("end", () => resolve());
+  });
+
+  await stdioTransport.close().catch(() => undefined);
+}

packages/server/src/server/voice-permission-policy.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, test } from "vitest";
+
+import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
+import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
+
+function buildRequest(partial: Partial<AgentPermissionRequest>): AgentPermissionRequest {
+  return {
+    id: "req-1",
+    provider: "codex",
+    name: "unknown",
+    kind: "tool",
+    ...partial,
+  };
+}
+
+describe("isVoicePermissionAllowed", () => {
+  test("allows speak tool", () => {
+    const result = isVoicePermissionAllowed(
+      buildRequest({ name: "speak" })
+    );
+    expect(result).toBe(true);
+  });
+
+  test("allows explicit MCP/paseo tool names", () => {
+    expect(
+      isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))
+    ).toBe(true);
+    expect(
+      isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))
+    ).toBe(true);
+  });
+
+  test("denies non-tool permission kinds", () => {
+    const result = isVoicePermissionAllowed(
+      buildRequest({ kind: "mode", name: "mcp__paseo__create_agent" })
+    );
+    expect(result).toBe(false);
+  });
+
+  test("allows codextool only when metadata references MCP/paseo", () => {
+    const allowed = isVoicePermissionAllowed(
+      buildRequest({
+        name: "codextool",
+        metadata: {
+          questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }],
+        },
+      })
+    );
+    expect(allowed).toBe(true);
+  });
+
+  test("denies codextool when metadata includes shell-like operations", () => {
+    const denied = isVoicePermissionAllowed(
+      buildRequest({
+        name: "codextool",
+        metadata: {
+          questions: [{ question: "Allow codextool to execute shell command?" }],
+        },
+      })
+    );
+    expect(denied).toBe(false);
+  });
+});

packages/server/src/server/voice-permission-policy.ts

@@ -0,0 +1,66 @@
+import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
+
+const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"];
+const DENIED_TOKEN_SET = [
+  "bash",
+  "shell",
+  "terminal",
+  "command",
+  "execute",
+  "edit",
+  "write",
+  "read",
+  "fetch",
+  "http",
+  "web",
+];
+
+function containsAny(text: string, tokens: readonly string[]): boolean {
+  return tokens.some((token) => text.includes(token));
+}
+
+function stringifyMetadata(metadata: unknown): string {
+  try {
+    return JSON.stringify(metadata ?? {})?.toLowerCase() ?? "";
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * Voice assistant policy: allow only MCP/paseo/speak tool requests.
+ * All non-tool permission requests are denied.
+ */
+export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean {
+  if (request.kind !== "tool") {
+    return false;
+  }
+
+  const normalizedName = request.name.trim().toLowerCase();
+  if (!normalizedName) {
+    return false;
+  }
+
+  if (normalizedName === "speak") {
+    return true;
+  }
+  if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) {
+    return true;
+  }
+
+  if (normalizedName !== "codextool") {
+    return false;
+  }
+
+  const metadataText = stringifyMetadata({
+    metadata: request.metadata ?? null,
+    input: request.input ?? null,
+  });
+  if (!metadataText) {
+    return false;
+  }
+
+  const mentionsAllowedTooling = containsAny(metadataText, ALLOWED_TOKEN_SET);
+  const mentionsDeniedTooling = containsAny(metadataText, DENIED_TOKEN_SET);
+  return mentionsAllowedTooling && !mentionsDeniedTooling;
+}

packages/server/src/server/voice-types.ts

@@ -0,0 +1,18 @@
+export type VoiceSpeakHandler = (params: {
+  text: string;
+  callerAgentId: string;
+  signal?: AbortSignal;
+}) => Promise<void>;
+
+export type VoiceCallerContext = {
+  childAgentDefaultLabels?: Record<string, string>;
+  lockedCwd?: string;
+  allowCustomCwd?: boolean;
+  enableVoiceTools?: boolean;
+};
+
+export type VoiceMcpStdioConfig = {
+  command: string;
+  baseArgs: string[];
+  env?: Record<string, string>;
+};

packages/server/src/server/websocket-server.ts

@@ -19,8 +19,13 @@ import { Session } from "./session.js";
 import type { AgentProvider } from "./agent/agent-sdk-types.js";
 import { PushTokenStore } from "./push/token-store.js";
 import { PushService } from "./push/push-service.js";
-import { VoiceConversationStore } from "./voice-conversation-store.js";
 import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
+import type { LocalSpeechModelId } from "./speech/providers/local/models.js";
+import type {
+  VoiceCallerContext,
+  VoiceMcpStdioConfig,
+  VoiceSpeakHandler,
+} from "./voice-types.js";
 
 export type AgentMcpTransportFactory = () => Promise<Transport>;
 
@@ -69,15 +74,26 @@ export class VoiceAssistantWebSocketServer {
   private readonly stt: SpeechToTextProvider | null;
   private readonly tts: TextToSpeechProvider | null;
   private readonly terminalManager: TerminalManager | null;
-  private readonly voiceConversationStore: VoiceConversationStore;
   private readonly dictation: {
     finalTimeoutMs?: number;
     stt?: SpeechToTextProvider | null;
+    localModels?: {
+      modelsDir: string;
+      defaultModelIds: LocalSpeechModelId[];
+    };
   } | null;
   private readonly voice: {
-    openrouterApiKey?: string | null;
+    voiceLlmProvider?: AgentProvider | null;
+    voiceLlmModeId?: string | null;
+    voiceLlmProviderExplicit?: boolean;
     voiceLlmModel?: string | null;
+    voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
   } | null;
+  private readonly voiceSpeakHandlers = new Map<
+    string,
+    VoiceSpeakHandler
+  >();
+  private readonly voiceCallerContexts = new Map<string, VoiceCallerContext>();
 
   constructor(
     server: HTTPServer,
@@ -92,12 +108,19 @@ export class VoiceAssistantWebSocketServer {
     speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
     terminalManager?: TerminalManager | null,
     voice?: {
-      openrouterApiKey?: string | null;
+      voiceLlmProvider?: AgentProvider | null;
+      voiceLlmModeId?: string | null;
+      voiceLlmProviderExplicit?: boolean;
       voiceLlmModel?: string | null;
+      voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
     },
     dictation?: {
       finalTimeoutMs?: number;
       stt?: SpeechToTextProvider | null;
+      localModels?: {
+        modelsDir: string;
+        defaultModelIds: LocalSpeechModelId[];
+      };
     }
   ) {
     this.logger = logger.child({ module: "websocket-server" });
@@ -110,9 +133,6 @@ export class VoiceAssistantWebSocketServer {
     this.stt = speech?.stt ?? null;
     this.tts = speech?.tts ?? null;
     this.terminalManager = terminalManager ?? null;
-    this.voiceConversationStore = new VoiceConversationStore(
-      join(paseoHome, "voice-conversations")
-    );
     this.voice = voice ?? null;
     this.dictation = dictation ?? null;
 
@@ -208,25 +228,38 @@ export class VoiceAssistantWebSocketServer {
     const clientId = `client-${++this.clientIdCounter}`;
     const connectionLogger = this.logger.child({ clientId });
 
-    const session = new Session(
+    const session = new Session({
       clientId,
-      (msg) => {
+      onMessage: (msg) => {
         this.sendToClient(ws, wrapSessionMessage(msg));
       },
-      connectionLogger.child({ module: "session" }),
-      this.downloadTokenStore,
-      this.pushTokenStore,
-      this.paseoHome,
-      this.agentManager,
-      this.agentStorage,
-      this.createAgentMcpTransport,
-      this.stt,
-      this.tts,
-      this.terminalManager,
-      this.voiceConversationStore,
-      this.voice ?? undefined,
-      this.dictation ?? undefined
-    );
+      logger: connectionLogger.child({ module: "session" }),
+      downloadTokenStore: this.downloadTokenStore,
+      pushTokenStore: this.pushTokenStore,
+      paseoHome: this.paseoHome,
+      agentManager: this.agentManager,
+      agentStorage: this.agentStorage,
+      createAgentMcpTransport: this.createAgentMcpTransport,
+      stt: this.stt,
+      tts: this.tts,
+      terminalManager: this.terminalManager,
+      voice: this.voice ?? undefined,
+      voiceBridge: {
+        registerVoiceSpeakHandler: (agentId, handler) => {
+          this.voiceSpeakHandlers.set(agentId, handler);
+        },
+        unregisterVoiceSpeakHandler: (agentId) => {
+          this.voiceSpeakHandlers.delete(agentId);
+        },
+        registerVoiceCallerContext: (agentId, context) => {
+          this.voiceCallerContexts.set(agentId, context);
+        },
+        unregisterVoiceCallerContext: (agentId) => {
+          this.voiceCallerContexts.delete(agentId);
+        },
+      },
+      dictation: this.dictation ?? undefined,
+    });
 
     this.sessions.set(ws, session);
 
@@ -263,6 +296,18 @@ export class VoiceAssistantWebSocketServer {
     });
   }
 
+  public resolveVoiceSpeakHandler(
+    callerAgentId: string
+  ): VoiceSpeakHandler | null {
+    return this.voiceSpeakHandlers.get(callerAgentId) ?? null;
+  }
+
+  public resolveVoiceCallerContext(
+    callerAgentId: string
+  ): VoiceCallerContext | null {
+    return this.voiceCallerContexts.get(callerAgentId) ?? null;
+  }
+
   private async detachSocket(
     ws: WebSocketLike,
     connectionLogger: pino.Logger,

packages/server/src/shared/messages.ts

@@ -294,11 +294,6 @@ export type AgentStreamEventPayload = z.infer<
 // Session Inbound Messages (Session receives these)
 // ============================================================================
 
-export const UserTextMessageSchema = z.object({
-  type: z.literal("user_text"),
-  text: z.string(),
-});
-
 export const VoiceAudioChunkMessageSchema = z.object({
   type: z.literal("voice_audio_chunk"),
   audio: z.string(), // base64 encoded
@@ -337,23 +332,6 @@ export const UnsubscribeAgentUpdatesMessageSchema = z.object({
   subscriptionId: z.string(),
 });
 
-export const LoadVoiceConversationRequestMessageSchema = z.object({
-  type: z.literal("load_voice_conversation_request"),
-  voiceConversationId: z.string(),
-  requestId: z.string(),
-});
-
-export const ListVoiceConversationsRequestMessageSchema = z.object({
-  type: z.literal("list_voice_conversations_request"),
-  requestId: z.string(),
-});
-
-export const DeleteVoiceConversationRequestMessageSchema = z.object({
-  type: z.literal("delete_voice_conversation_request"),
-  voiceConversationId: z.string(),
-  requestId: z.string(),
-});
-
 export const DeleteAgentRequestMessageSchema = z.object({
   type: z.literal("delete_agent_request"),
   agentId: z.string(),
@@ -366,10 +344,10 @@ export const ArchiveAgentRequestMessageSchema = z.object({
   requestId: z.string(),
 });
 
-export const SetVoiceConversationMessageSchema = z.object({
-  type: z.literal("set_voice_conversation"),
+export const SetVoiceModeMessageSchema = z.object({
+  type: z.literal("set_voice_mode"),
   enabled: z.boolean(),
-  voiceConversationId: z.string().optional(),
+  voiceAgentId: z.string().optional(),
 });
 
 export const SendAgentMessageSchema = z.object({
@@ -876,7 +854,6 @@ export const KillTerminalRequestSchema = z.object({
 });
 
 export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
-  UserTextMessageSchema,
   VoiceAudioChunkMessageSchema,
   AbortRequestMessageSchema,
   AudioPlayedMessageSchema,
@@ -884,12 +861,9 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
   FetchAgentRequestMessageSchema,
   SubscribeAgentUpdatesMessageSchema,
   UnsubscribeAgentUpdatesMessageSchema,
-  LoadVoiceConversationRequestMessageSchema,
-  ListVoiceConversationsRequestMessageSchema,
-  DeleteVoiceConversationRequestMessageSchema,
   DeleteAgentRequestMessageSchema,
   ArchiveAgentRequestMessageSchema,
-  SetVoiceConversationMessageSchema,
+  SetVoiceModeMessageSchema,
   SendAgentMessageRequestSchema,
   WaitForFinishRequestSchema,
   DictationStreamStartMessageSchema,
@@ -1127,15 +1101,6 @@ export const ArtifactMessageSchema = z.object({
   }),
 });
 
-export const VoiceConversationLoadedMessageSchema = z.object({
-  type: z.literal("voice_conversation_loaded"),
-  payload: z.object({
-    voiceConversationId: z.string(),
-    messageCount: z.number(),
-    requestId: z.string(),
-  }),
-});
-
 export const AgentUpdateMessageSchema = z.object({
   type: z.literal("agent_update"),
   payload: z.discriminatedUnion("kind", [
@@ -1225,30 +1190,6 @@ export const WaitForFinishResponseMessageSchema = z.object({
   }),
 });
 
-export const ListVoiceConversationsResponseMessageSchema = z.object({
-  type: z.literal("list_voice_conversations_response"),
-  payload: z.object({
-    conversations: z.array(
-      z.object({
-        id: z.string(),
-        lastUpdated: z.string(),
-        messageCount: z.number(),
-      })
-    ),
-    requestId: z.string(),
-  }),
-});
-
-export const DeleteVoiceConversationResponseMessageSchema = z.object({
-  type: z.literal("delete_voice_conversation_response"),
-  payload: z.object({
-    voiceConversationId: z.string(),
-    success: z.boolean(),
-    error: z.string().optional(),
-    requestId: z.string(),
-  }),
-});
-
 export const AgentPermissionRequestMessageSchema = z.object({
   type: z.literal("agent_permission_request"),
   payload: z.object({
@@ -1679,7 +1620,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
   RpcErrorMessageSchema,
   InitializeAgentResponseMessageSchema,
   ArtifactMessageSchema,
-  VoiceConversationLoadedMessageSchema,
   AgentUpdateMessageSchema,
   AgentStreamMessageSchema,
   AgentStreamSnapshotMessageSchema,
@@ -1691,8 +1631,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
   SetAgentModelResponseMessageSchema,
   SetAgentThinkingResponseMessageSchema,
   WaitForFinishResponseMessageSchema,
-  ListVoiceConversationsResponseMessageSchema,
-  DeleteVoiceConversationResponseMessageSchema,
   AgentPermissionRequestMessageSchema,
   AgentPermissionResolvedMessageSchema,
   AgentDeletedMessageSchema,
@@ -1737,9 +1675,6 @@ export type TranscriptionResultMessage = z.infer<typeof TranscriptionResultMessa
 export type StatusMessage = z.infer<typeof StatusMessageSchema>;
 export type RpcErrorMessage = z.infer<typeof RpcErrorMessageSchema>;
 export type ArtifactMessage = z.infer<typeof ArtifactMessageSchema>;
-export type VoiceConversationLoadedMessage = z.infer<
-  typeof VoiceConversationLoadedMessageSchema
->;
 export type AgentUpdateMessage = z.infer<typeof AgentUpdateMessageSchema>;
 export type AgentStreamMessage = z.infer<typeof AgentStreamMessageSchema>;
 export type AgentStreamSnapshotMessage = z.infer<
@@ -1758,12 +1693,6 @@ export type SendAgentMessageResponseMessage = z.infer<
 export type WaitForFinishResponseMessage = z.infer<
   typeof WaitForFinishResponseMessageSchema
 >;
-export type ListVoiceConversationsResponseMessage = z.infer<
-  typeof ListVoiceConversationsResponseMessageSchema
->;
-export type DeleteVoiceConversationResponseMessage = z.infer<
-  typeof DeleteVoiceConversationResponseMessageSchema
->;
 export type AgentPermissionRequestMessage = z.infer<typeof AgentPermissionRequestMessageSchema>;
 export type AgentPermissionResolvedMessage = z.infer<typeof AgentPermissionResolvedMessageSchema>;
 export type AgentDeletedMessage = z.infer<typeof AgentDeletedMessageSchema>;
@@ -1778,7 +1707,6 @@ export type InitializeAgentResponseMessage = z.infer<typeof InitializeAgentRespo
 export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
 
 // Type exports for inbound message types
-export type UserTextMessage = z.infer<typeof UserTextMessageSchema>;
 export type VoiceAudioChunkMessage = z.infer<typeof VoiceAudioChunkMessageSchema>;
 export type FetchAgentsRequestMessage = z.infer<typeof FetchAgentsRequestMessageSchema>;
 export type FetchAgentRequestMessage = z.infer<typeof FetchAgentRequestMessageSchema>;

packages/server/src/test-utils/vitest-setup.ts

@@ -1,7 +1,8 @@
 import path from "node:path";
 import dotenv from "dotenv";
 
-// Load repo-root .env for integration/E2E tests (OpenAI, etc.)
+// Load package-local .env.test first for integration/E2E credentials, then repo-root .env fallback.
+dotenv.config({ path: path.resolve(process.cwd(), ".env.test"), override: true });
 dotenv.config({ path: path.resolve(process.cwd(), "../.env") });
 
 process.env.GIT_TERMINAL_PROMPT = "0";

packages/server/src/utils/tool-call-parsers.ts

@@ -28,6 +28,7 @@ const PrincipalParamSchema = z.union([
   z.object({ pattern: z.string() }).transform((d) => ({ type: "text" as const, value: d.pattern })),
   z.object({ query: z.string() }).transform((d) => ({ type: "text" as const, value: d.query })),
   z.object({ url: z.string() }).transform((d) => ({ type: "text" as const, value: d.url })),
+  z.object({ text: z.string() }).transform((d) => ({ type: "text" as const, value: d.text })),
   // Files array (Codex apply_patch)
   z.object({ files: z.array(FileEntrySchema).nonempty() }).transform((d) => ({ type: "path" as const, value: d.files[0].path })),
   // TodoWrite - show in_progress item or count