From 8c70b3d86bea4735004775c084175587f5342081 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 21:27:50 +0700 Subject: [PATCH] voice: harden local voice stack and enforce UUID agent ids --- package-lock.json | 12 - packages/cli/src/cli.ts | 4 +- packages/cli/src/commands/voice-mcp-bridge.ts | 4 +- packages/server/package.json | 5 +- .../src/server/agent/activity-curator.test.ts | 16 ++ .../src/server/agent/activity-curator.ts | 9 +- .../src/server/agent/agent-manager.test.ts | 111 ++++++-- .../server/src/server/agent/agent-manager.ts | 43 ++- .../server/src/server/agent/mcp-server.ts | 13 +- .../server/src/server/bootstrap.smoke.test.ts | 4 +- packages/server/src/server/bootstrap.ts | 16 +- .../src/server/daemon-client.e2e.test.ts | 9 +- packages/server/src/server/session.ts | 253 ++++++++++-------- .../local/sherpa/speech-download.e2e.test.ts | 3 +- .../server/speech/speech-config-resolver.ts | 12 +- .../src/server/speech/speech-runtime.ts | 12 +- .../server/src/server/speech/speech-types.ts | 10 + .../src/server/voice-local-agent.e2e.test.ts | 3 +- .../server/src/server/voice-mcp-bridge.ts | 74 +++-- .../server/voice-permission-policy.test.ts | 63 +++++ .../src/server/voice-permission-policy.ts | 66 +++++ packages/server/src/server/voice-types.ts | 18 ++ .../server/src/server/websocket-server.ts | 63 ++--- 23 files changed, 574 insertions(+), 249 deletions(-) create mode 100644 packages/server/src/server/speech/speech-types.ts create mode 100644 packages/server/src/server/voice-permission-policy.test.ts create mode 100644 packages/server/src/server/voice-permission-policy.ts create mode 100644 packages/server/src/server/voice-types.ts diff --git a/package-lock.json b/package-lock.json index 4ec5f45a0..3c6ed89a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4688,17 +4688,6 @@ "version": "1.1.12", "license": "MIT" }, - "node_modules/@openrouter/ai-sdk-provider": { - "version": "1.2.0", - "license": "Apache-2.0", - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "ai": "^5.0.0", - "zod": "^3.24.1 || ^v4" - } - }, "node_modules/@opentelemetry/api": { "version": "1.9.0", "license": "Apache-2.0", @@ -20286,7 +20275,6 @@ "@lezer/python": "^1.1.18", "@modelcontextprotocol/sdk": "^1.20.1", "@opencode-ai/sdk": "^1.1.12", - "@openrouter/ai-sdk-provider": "^1.2.0", "@sctg/sentencepiece-js": "^1.1.0", "@xterm/headless": "^6.0.0", "ai": "^5.0.76", diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index ee98ae87d..b5f2d6c03 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -144,10 +144,10 @@ export function createCli(): Command { // Internal voice MCP stdio bridge command (hidden). program - .command('__paseo_voice_mcp_bridge') + .command('__paseo_voice_mcp_bridge', { hidden: true }) .description('Internal voice MCP bridge command') - .argument('') .requiredOption('--socket ') + .requiredOption('--caller-agent-id ') .action(runVoiceMcpBridgeCommand) return program diff --git a/packages/cli/src/commands/voice-mcp-bridge.ts b/packages/cli/src/commands/voice-mcp-bridge.ts index ddbb68bb8..e50887fa7 100644 --- a/packages/cli/src/commands/voice-mcp-bridge.ts +++ b/packages/cli/src/commands/voice-mcp-bridge.ts @@ -2,16 +2,16 @@ import { runVoiceMcpBridgeCli } from "@getpaseo/server"; type VoiceBridgeOptions = { socket: string; + callerAgentId: string; }; export async function runVoiceMcpBridgeCommand( - callerAgentId: string, options: VoiceBridgeOptions ): Promise { await runVoiceMcpBridgeCli([ "--socket", options.socket, "--caller-agent-id", - callerAgentId, + options.callerAgentId, ]); } diff --git a/packages/server/package.json b/packages/server/package.json index 88293bf86..dc935208e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -36,7 +36,7 @@ "@lezer/python": "^1.1.18", "@modelcontextprotocol/sdk": "^1.20.1", "@opencode-ai/sdk": "^1.1.12", - "@openrouter/ai-sdk-provider": "^1.2.0", + "@sctg/sentencepiece-js": "^1.1.0", "@xterm/headless": "^6.0.0", "ai": "^5.0.76", "ajv": "^8.17.1", @@ -45,13 +45,12 @@ "express-basic-auth": "^1.2.1", "mnemonic-id": "^3.2.7", "node-pty": "^1.0.0", + "onnxruntime-node": "^1.23.0", "openai": "^4.20.0", "pino": "^10.2.0", "pino-pretty": "^13.1.3", "playwright": "^1.56.1", "qrcode": "^1.5.4", - "@sctg/sentencepiece-js": "^1.1.0", - "onnxruntime-node": "^1.23.0", "sherpa-onnx": "^1.12.23", "sherpa-onnx-node": "^1.12.23", "strip-ansi": "^7.1.2", diff --git a/packages/server/src/server/agent/activity-curator.test.ts b/packages/server/src/server/agent/activity-curator.test.ts index dad730222..781d1342b 100644 --- a/packages/server/src/server/agent/activity-curator.test.ts +++ b/packages/server/src/server/agent/activity-curator.test.ts @@ -66,6 +66,22 @@ describe("curateAgentActivity", () => { expect(result).toBe("[ListFiles]"); }); + test("does not treat generic double-underscore tool names as MCP calls", () => { + const timeline: AgentTimelineItem[] = [ + { + type: "tool_call", + callId: "call-1", + name: "custom__tool", + input: {}, + status: "completed", + }, + ]; + + const result = curateAgentActivity(timeline); + + expect(result).toBe("[custom__tool]"); + }); + test("serializes todo items as [Tasks]", () => { const timeline: AgentTimelineItem[] = [ { diff --git a/packages/server/src/server/agent/activity-curator.ts b/packages/server/src/server/agent/activity-curator.ts index 5c8425c33..8370b0d6a 100644 --- a/packages/server/src/server/agent/activity-curator.ts +++ b/packages/server/src/server/agent/activity-curator.ts @@ -29,10 +29,11 @@ function flushBuffers(lines: string[], buffers: { message: string; thought: stri function isLikelyMcpToolCall(name: string): boolean { const normalized = name.toLowerCase(); return ( - normalized.includes("mcp") || - normalized.includes("paseo") || - normalized.includes("__") || - normalized === "speak" + normalized === "speak" || + normalized.startsWith("mcp") || + normalized.includes("mcp__") || + normalized.startsWith("paseo") || + normalized.includes("paseo__") ); } diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 6e69d85fc..d7302ef77 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -118,7 +118,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "agent-without-model", + idFactory: () => "00000000-0000-4000-8000-000000000101", }); const snapshot = await manager.createAgent({ @@ -149,7 +149,7 @@ describe("AgentManager", () => { ).rejects.toThrow("Working directory does not exist"); }); - test("createAgent persists provided title before returning", async () => { + test("createAgent fails when generated agent ID is not a UUID", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); @@ -159,7 +159,52 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "agent-with-title", + idFactory: () => "not-a-uuid", + }); + + await expect( + manager.createAgent({ + provider: "codex", + cwd: workdir, + }) + ).rejects.toThrow("createAgent: agentId must be a UUID"); + }); + + test("createAgent fails when explicit agent ID is not a UUID", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); + const storagePath = join(workdir, "agents"); + const storage = new AgentStorage(storagePath, logger); + const manager = new AgentManager({ + clients: { + codex: new TestAgentClient(), + }, + registry: storage, + logger, + }); + + await expect( + manager.createAgent( + { + provider: "codex", + cwd: workdir, + }, + "not-a-uuid" + ) + ).rejects.toThrow("createAgent: agentId must be a UUID"); + }); + + test("createAgent persists provided title before returning", async () => { + const agentId = "00000000-0000-4000-8000-000000000102"; + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); + const storagePath = join(workdir, "agents"); + const storage = new AgentStorage(storagePath, logger); + const manager = new AgentManager({ + clients: { + codex: new TestAgentClient(), + }, + registry: storage, + logger, + idFactory: () => agentId, }); const snapshot = await manager.createAgent({ @@ -168,12 +213,12 @@ describe("AgentManager", () => { title: "Fix Login Bug", }); - expect(snapshot.id).toBe("agent-with-title"); + expect(snapshot.id).toBe(agentId); expect(snapshot.lifecycle).toBe("idle"); - const persisted = await storage.get("agent-with-title"); + const persisted = await storage.get(agentId); expect(persisted?.title).toBe("Fix Login Bug"); - expect(persisted?.id).toBe("agent-with-title"); + expect(persisted?.id).toBe(agentId); }); test("createAgent populates runtimeInfo after session creation", async () => { @@ -186,7 +231,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "agent-with-runtime-info", + idFactory: () => "00000000-0000-4000-8000-000000000103", }); const snapshot = await manager.createAgent({ @@ -211,7 +256,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "agent-with-run-runtime", + idFactory: () => "00000000-0000-4000-8000-000000000104", }); const snapshot = await manager.createAgent({ @@ -231,6 +276,10 @@ describe("AgentManager", () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); + const generatedAgentIds = [ + "00000000-0000-4000-8000-000000000105", + "00000000-0000-4000-8000-000000000106", + ]; let agentCounter = 0; const manager = new AgentManager({ clients: { @@ -238,7 +287,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => `agent-${agentCounter++}`, + idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(), }); // Create a normal agent @@ -262,6 +311,7 @@ describe("AgentManager", () => { }); test("getAgent returns internal agents by ID", async () => { + const internalAgentId = "00000000-0000-4000-8000-000000000107"; const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); @@ -271,7 +321,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "internal-agent", + idFactory: () => internalAgentId, }); await manager.createAgent({ @@ -281,7 +331,7 @@ describe("AgentManager", () => { internal: true, }); - const agent = manager.getAgent("internal-agent"); + const agent = manager.getAgent(internalAgentId); expect(agent).not.toBeNull(); expect(agent?.internal).toBe(true); }); @@ -290,6 +340,10 @@ describe("AgentManager", () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); + const generatedAgentIds = [ + "00000000-0000-4000-8000-000000000108", + "00000000-0000-4000-8000-000000000109", + ]; let agentCounter = 0; const manager = new AgentManager({ clients: { @@ -297,7 +351,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => `agent-${agentCounter++}`, + idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(), }); const receivedEvents: string[] = []; @@ -323,11 +377,12 @@ describe("AgentManager", () => { }); // Should only have events from the normal agent - expect(receivedEvents.filter((id) => id === "agent-0").length).toBeGreaterThan(0); - expect(receivedEvents.filter((id) => id === "agent-1").length).toBe(0); + expect(receivedEvents.filter((id) => id === generatedAgentIds[0]).length).toBeGreaterThan(0); + expect(receivedEvents.filter((id) => id === generatedAgentIds[1]).length).toBe(0); }); test("subscribe emits state events for internal agents when subscribed by agentId", async () => { + const internalAgentId = "00000000-0000-4000-8000-000000000110"; const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); @@ -337,7 +392,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "internal-agent", + idFactory: () => internalAgentId, }); const receivedEvents: string[] = []; @@ -348,7 +403,7 @@ describe("AgentManager", () => { receivedEvents.push(event.agent.id); } }, - { agentId: "internal-agent", replayState: false } + { agentId: internalAgentId, replayState: false } ); await manager.createAgent({ @@ -359,10 +414,26 @@ describe("AgentManager", () => { }); // Should receive events when subscribed by specific agentId - expect(receivedEvents.filter((id) => id === "internal-agent").length).toBeGreaterThan(0); + expect(receivedEvents.filter((id) => id === internalAgentId).length).toBeGreaterThan(0); + }); + + test("subscribe fails when filter agentId is not a UUID", () => { + const manager = new AgentManager({ + clients: { + codex: new TestAgentClient(), + }, + logger, + }); + + expect(() => + manager.subscribe(() => {}, { + agentId: "invalid-agent-id", + }) + ).toThrow("subscribe: agentId must be a UUID"); }); test("onAgentAttention is not called for internal agents", async () => { + const internalAgentId = "00000000-0000-4000-8000-000000000111"; const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); @@ -373,7 +444,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "internal-agent", + idFactory: () => internalAgentId, onAgentAttention: ({ agentId }) => { attentionCalls.push(agentId); }, @@ -481,7 +552,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "plan-mode-agent", + idFactory: () => "00000000-0000-4000-8000-000000000112", }); // Create agent in plan mode @@ -608,7 +679,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "close-race-agent", + idFactory: () => "00000000-0000-4000-8000-000000000113", }); const snapshot = await manager.createAgent({ diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 76ec69fc4..efb939c15 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -6,6 +6,7 @@ import { type AgentLifecycleStatus, } from "../../shared/agent-lifecycle.js"; import type { Logger } from "pino"; +import { z } from "zod"; import type { AgentCapabilityFlags, @@ -183,6 +184,7 @@ const BUSY_STATUSES: AgentLifecycleStatus[] = [ "initializing", "running", ]; +const AgentIdSchema = z.string().uuid(); function isAgentBusy(status: AgentLifecycleStatus): boolean { return BUSY_STATUSES.includes(status); @@ -202,6 +204,14 @@ function createAbortError( return Object.assign(new Error(message), { name: "AbortError" }); } +function validateAgentId(agentId: string, source: string): string { + const result = AgentIdSchema.safeParse(agentId); + if (!result.success) { + throw new Error(`${source}: agentId must be a UUID`); + } + return result.data; +} + export class AgentManager { private readonly clients = new Map(); private readonly agents = new Map(); @@ -239,9 +249,13 @@ export class AgentManager { } subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void { + const targetAgentId = + options?.agentId == null + ? null + : validateAgentId(options.agentId, "subscribe"); const record: SubscriptionRecord = { callback, - agentId: options?.agentId ?? null, + agentId: targetAgentId, }; this.subscribers.add(record); @@ -335,7 +349,10 @@ export class AgentManager { options?: { labels?: Record } ): Promise { // Generate agent ID early so we can use it in MCP config - const resolvedAgentId = agentId ?? this.idFactory(); + const resolvedAgentId = validateAgentId( + agentId ?? this.idFactory(), + "createAgent" + ); const normalizedConfig = await this.normalizeConfig(config, { labels: options?.labels, agentId: resolvedAgentId, @@ -365,6 +382,10 @@ export class AgentManager { labels?: Record; } ): Promise { + const resolvedAgentId = validateAgentId( + agentId ?? this.idFactory(), + "resumeAgent" + ); const metadata = (handle.metadata ?? {}) as Partial; const mergedConfig = { ...metadata, @@ -381,7 +402,7 @@ export class AgentManager { return this.registerSession( session, normalizedConfig, - agentId ?? this.idFactory(), + resolvedAgentId, options ); } @@ -982,13 +1003,14 @@ export class AgentManager { labels?: Record; } ): Promise { - if (this.agents.has(agentId)) { - throw new Error(`Agent with id ${agentId} already exists`); + const resolvedAgentId = validateAgentId(agentId, "registerSession"); + if (this.agents.has(resolvedAgentId)) { + throw new Error(`Agent with id ${resolvedAgentId} already exists`); } const now = new Date(); const managed = { - id: agentId, + id: resolvedAgentId, provider: config.provider, cwd: config.cwd, session, @@ -1011,9 +1033,9 @@ export class AgentManager { labels: options?.labels ?? {}, } as ActiveManagedAgent; - this.agents.set(agentId, managed); + this.agents.set(resolvedAgentId, managed); // Initialize previousStatus to track transitions - this.previousStatuses.set(agentId, managed.lifecycle); + this.previousStatuses.set(resolvedAgentId, managed.lifecycle); await this.refreshRuntimeInfo(managed); await this.persistSnapshot(managed, { title: config.title ?? null, @@ -1353,9 +1375,10 @@ export class AgentManager { } private requireAgent(id: string): ActiveManagedAgent { - const agent = this.agents.get(id); + const normalizedId = validateAgentId(id, "requireAgent"); + const agent = this.agents.get(normalizedId); if (!agent) { - throw new Error(`Unknown agent '${id}'`); + throw new Error(`Unknown agent '${normalizedId}'`); } return agent; } diff --git a/packages/server/src/server/agent/mcp-server.ts b/packages/server/src/server/agent/mcp-server.ts index 83d09a9ab..3c83aae76 100644 --- a/packages/server/src/server/agent/mcp-server.ts +++ b/packages/server/src/server/agent/mcp-server.ts @@ -28,6 +28,10 @@ import { AgentStorage } from "./agent-storage.js"; import { createWorktree } from "../../utils/worktree.js"; import { WaitForAgentTracker } from "./wait-for-agent-tracker.js"; import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js"; +import type { + VoiceCallerContext, + VoiceSpeakHandler, +} from "../voice-types.js"; export interface AgentMcpServerOptions { agentManager: AgentManager; @@ -44,15 +48,10 @@ export interface AgentMcpServerOptions { */ resolveSpeakHandler?: ( callerAgentId: string - ) => ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise) | null; + ) => VoiceSpeakHandler | null; resolveCallerContext?: ( callerAgentId: string - ) => { - childAgentDefaultLabels?: Record; - lockedCwd?: string; - allowCustomCwd?: boolean; - enableVoiceTools?: boolean; - } | null; + ) => VoiceCallerContext | null; enableVoiceTools?: boolean; logger: Logger; } diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 82e82984a..ac19a9d90 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -9,9 +9,9 @@ import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js"; import { createTestAgentClients } from "./test-utils/fake-agent-client.js"; describe("paseo daemon bootstrap", () => { - test.runIf(Boolean(process.env.OPENAI_API_KEY))("starts and serves health endpoint", async () => { + test("starts and serves health endpoint", async () => { const daemonHandle = await createTestPaseoDaemon({ - openai: { apiKey: process.env.OPENAI_API_KEY! }, + openai: { apiKey: "test-openai-api-key" }, speech: { dictationSttProvider: "openai", voiceSttProvider: "openai", diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 04cfb27b6..12d6d61ac 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -73,24 +73,32 @@ type AgentMcpTransportMap = Map; function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } { const explicit = process.env.PASEO_BIN_PATH?.trim(); if (explicit) { - return { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] }; + const resolved = { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] }; + logger.info({ source: "PASEO_BIN_PATH", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command"); + return resolved; } const argv1 = process.argv[1]?.trim(); if (!argv1) { logger.warn("Could not resolve argv[1] for voice MCP bridge; falling back to 'paseo'"); - return { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] }; + const resolved = { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] }; + logger.info({ source: "fallback", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command"); + return resolved; } const base = path.basename(argv1).toLowerCase(); if (base.includes("tsx") && process.argv[2]) { - return { + const resolved = { command: process.execPath, baseArgs: [argv1, process.argv[2], "__paseo_voice_mcp_bridge"], }; + logger.info({ source: "tsx", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command"); + return resolved; } - return { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] }; + const resolved = { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] }; + logger.info({ source: "argv", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command"); + return resolved; } export type PaseoOpenAIConfig = { diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index 3941ecb0f..f6fe778ec 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs"; import { tmpdir, homedir } from "node:os"; import path from "node:path"; import { execSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; import { createDaemonTestContext, @@ -129,7 +130,7 @@ describe("daemon client E2E", () => { test("handles session actions", async () => { expect(ctx.client.isConnected).toBe(true); - const voiceConversationId = `voice-${Date.now()}`; + const voiceConversationId = randomUUID(); const loadResult = await ctx.client.loadVoiceConversation(voiceConversationId); expect(loadResult.voiceConversationId).toBe(voiceConversationId); expect(typeof loadResult.messageCount).toBe("number"); @@ -140,7 +141,7 @@ describe("daemon client E2E", () => { const listResult = await ctx.client.listVoiceConversations(); expect(Array.isArray(listResult.conversations)).toBe(true); - const missingId = `missing-${Date.now()}`; + const missingId = randomUUID(); const deleteResult = await ctx.client.deleteVoiceConversation(missingId); expect(deleteResult.voiceConversationId).toBe(missingId); expect(deleteResult.success).toBe(false); @@ -577,7 +578,7 @@ describe("daemon client E2E", () => { ); test( - "does not process non-voice LLM turns via OpenRouter", + "does not process non-voice turns through the voice agent path", async () => { await ctx.client.setVoiceConversation(false); @@ -628,7 +629,7 @@ describe("daemon client E2E", () => { speechTest( "voice mode buffers audio until isLast and emits transcription_result", async () => { - await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`); + await ctx.client.setVoiceConversation(true, randomUUID()); const transcription = waitForSignal(30_000, (resolve) => { const unsubscribe = ctx.client.on("transcription_result", (message) => { diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index ff4c87e1d..7f2cc2e4f 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -37,20 +37,13 @@ import { } from "./persistence-hooks.js"; import { experimental_createMCPClient } from "ai"; import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js"; +import type { + VoiceCallerContext, + VoiceMcpStdioConfig, + VoiceSpeakHandler, +} from "./voice-types.js"; export type AgentMcpTransportFactory = () => Promise; -type VoiceSpeakHandler = (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise; -type VoiceCallerContext = { - childAgentDefaultLabels?: Record; - lockedCwd?: string; - allowCustomCwd?: boolean; - enableVoiceTools?: boolean; -}; -type VoiceMcpStdioConfig = { - command: string; - baseArgs: string[]; - env?: Record; -}; import { buildProviderRegistry } from "./agent/provider-registry.js"; import { AgentManager } from "./agent/agent-manager.js"; import type { ManagedAgent } from "./agent/agent-manager.js"; @@ -73,6 +66,7 @@ import type { } from "./agent/agent-sdk-types.js"; import { AgentStorage, type StoredAgentRecord } from "./agent/agent-storage.js"; import { isValidAgentProvider, AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; +import { isVoicePermissionAllowed } from "./voice-permission-policy.js"; import { listDirectoryEntries, readExplorerFile, @@ -188,6 +182,7 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round( PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS ); const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/; +const VoiceConversationIdSchema = z.string().uuid(); interface AudioBufferState { chunks: Buffer[]; @@ -196,6 +191,42 @@ interface AudioBufferState { totalPCMBytes: number; } +export type SessionOptions = { + clientId: string; + onMessage: (msg: SessionOutboundMessage) => void; + logger: pino.Logger; + downloadTokenStore: DownloadTokenStore; + pushTokenStore: PushTokenStore; + paseoHome: string; + agentManager: AgentManager; + agentStorage: AgentStorage; + createAgentMcpTransport: AgentMcpTransportFactory; + stt: SpeechToTextProvider | null; + tts: TextToSpeechProvider | null; + terminalManager: TerminalManager | null; + voice?: { + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; + voiceLlmProviderExplicit?: boolean; + voiceLlmModel?: string | null; + voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; + }; + voiceBridge?: { + registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void; + unregisterVoiceSpeakHandler?: (agentId: string) => void; + registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void; + unregisterVoiceCallerContext?: (agentId: string) => void; + }; + dictation?: { + finalTimeoutMs?: number; + stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + }; + }; +}; + function convertPCMToWavBuffer( pcmBuffer: Buffer, sampleRate: number, @@ -353,41 +384,24 @@ export class Session { private readonly unregisterVoiceCallerContext?: (agentId: string) => void; private voiceAssistantAgentId: string | null = null; - constructor( - clientId: string, - onMessage: (msg: SessionOutboundMessage) => void, - logger: pino.Logger, - downloadTokenStore: DownloadTokenStore, - pushTokenStore: PushTokenStore, - paseoHome: string, - agentManager: AgentManager, - agentStorage: AgentStorage, - createAgentMcpTransport: AgentMcpTransportFactory, - stt: SpeechToTextProvider | null, - tts: TextToSpeechProvider | null, - terminalManager: TerminalManager | null, - voice?: { - voiceLlmProvider?: AgentProvider | null; - voiceLlmModeId?: string | null; - voiceLlmProviderExplicit?: boolean; - voiceLlmModel?: string | null; - voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; - }, - voiceBridge?: { - registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void; - unregisterVoiceSpeakHandler?: (agentId: string) => void; - registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void; - unregisterVoiceCallerContext?: (agentId: string) => void; - }, - dictation?: { - finalTimeoutMs?: number; - stt?: SpeechToTextProvider | null; - localModels?: { - modelsDir: string; - defaultModelIds: SherpaOnnxModelId[]; - }; - } - ) { + constructor(options: SessionOptions) { + const { + clientId, + onMessage, + logger, + downloadTokenStore, + pushTokenStore, + paseoHome, + agentManager, + agentStorage, + createAgentMcpTransport, + stt, + tts, + terminalManager, + voice, + voiceBridge, + dictation, + } = options; this.clientId = clientId; this.sessionId = uuidv4(); this.onMessage = onMessage; @@ -1148,12 +1162,16 @@ export class Session { voiceConversationId: string, requestId: string ): Promise { - this.voiceAssistantAgentId = voiceConversationId; + const normalizedVoiceConversationId = this.parseVoiceConversationId( + voiceConversationId, + "load_voice_conversation_request" + ); + this.voiceAssistantAgentId = normalizedVoiceConversationId; this.emit({ type: "voice_conversation_loaded", payload: { - voiceConversationId, + voiceConversationId: normalizedVoiceConversationId, messageCount: 0, requestId, }, @@ -1216,13 +1234,17 @@ export class Session { voiceConversationId: string, requestId: string ): Promise { + const normalizedVoiceConversationId = this.parseVoiceConversationId( + voiceConversationId, + "delete_voice_conversation_request" + ); try { - const record = await this.agentStorage.get(voiceConversationId); + const record = await this.agentStorage.get(normalizedVoiceConversationId); if (!record || record.labels?.surface !== "voice") { this.emit({ type: "delete_voice_conversation_response", payload: { - voiceConversationId, + voiceConversationId: normalizedVoiceConversationId, success: false, error: "Voice conversation not found", requestId, @@ -1231,32 +1253,32 @@ export class Session { return; } - const live = this.agentManager.getAgent(voiceConversationId); + const live = this.agentManager.getAgent(normalizedVoiceConversationId); if (live) { - await this.agentManager.closeAgent(voiceConversationId); + await this.agentManager.closeAgent(normalizedVoiceConversationId); } - await this.agentStorage.remove(voiceConversationId); + await this.agentStorage.remove(normalizedVoiceConversationId); this.emit({ type: "delete_voice_conversation_response", payload: { - voiceConversationId, + voiceConversationId: normalizedVoiceConversationId, success: true, requestId, }, }); this.sessionLogger.info( - { voiceConversationId }, - `Deleted voice conversation ${voiceConversationId}` + { voiceConversationId: normalizedVoiceConversationId }, + `Deleted voice conversation ${normalizedVoiceConversationId}` ); } catch (error: any) { this.sessionLogger.error( - { err: error, voiceConversationId }, - `Failed to delete voice conversation ${voiceConversationId}` + { err: error, voiceConversationId: normalizedVoiceConversationId }, + `Failed to delete voice conversation ${normalizedVoiceConversationId}` ); this.emit({ type: "delete_voice_conversation_response", payload: { - voiceConversationId, + voiceConversationId: normalizedVoiceConversationId, success: false, error: error.message, requestId, @@ -1393,15 +1415,22 @@ export class Session { voiceConversationId?: string ): Promise { if (enabled) { - if (!voiceConversationId || voiceConversationId.trim().length === 0) { - this.sessionLogger.warn("set_voice_conversation missing voiceConversationId; ignoring"); - return; - } + const candidateVoiceConversationId = + voiceConversationId && voiceConversationId.trim().length > 0 + ? voiceConversationId + : uuidv4(); + const normalizedVoiceConversationId = this.parseVoiceConversationId( + candidateVoiceConversationId, + "set_voice_conversation" + ); this.isVoiceMode = true; - this.voiceAssistantAgentId = voiceConversationId; + this.voiceAssistantAgentId = normalizedVoiceConversationId; this.sessionLogger.info( - { voiceAssistantAgentId: this.voiceAssistantAgentId }, + { + voiceAssistantAgentId: this.voiceAssistantAgentId, + generatedConversationId: !voiceConversationId, + }, "Voice conversation enabled (agent-backed)" ); return; @@ -1414,6 +1443,14 @@ export class Session { ); } + private parseVoiceConversationId(rawId: string, source: string): string { + const parsed = VoiceConversationIdSchema.safeParse(rawId.trim()); + if (!parsed.success) { + throw new Error(`${source}: voiceConversationId must be a UUID`); + } + return parsed.data; + } + /** * Handle text message to agent (with optional image attachments) */ @@ -4259,7 +4296,7 @@ export class Session { }); // Process through LLM (voice path is agent-backed only) - this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, text); + this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, text); await this.currentStreamPromise; } @@ -4528,7 +4565,7 @@ export class Session { // Set phase to LLM and process (TTS enabled in voice mode for voice conversations) this.clearSpeechInProgress("transcription complete"); this.setPhase("llm"); - this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, result.text); + this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, result.text); await this.currentStreamPromise; this.setPhase("idle"); } catch (error: any) { @@ -4601,30 +4638,47 @@ export class Session { }, }); this.voiceAssistantAgentId = created.id; + let speakHandlerRegistered = false; + let callerContextRegistered = false; - this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => { - const abortSignal = signal ?? this.abortController.signal; - await this.ttsManager.generateAndWaitForPlayback( - text, - (msg) => this.emit(msg), - abortSignal, - true - ); - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "assistant", - content: text, - }, + try { + this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => { + const abortSignal = signal ?? this.abortController.signal; + await this.ttsManager.generateAndWaitForPlayback( + text, + (msg) => this.emit(msg), + abortSignal, + true + ); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "assistant", + content: text, + }, + }); }); - }); - this.registerVoiceCallerContext?.(created.id, { - childAgentDefaultLabels: { ui: "true" }, - allowCustomCwd: true, - enableVoiceTools: true, - }); + speakHandlerRegistered = true; + + this.registerVoiceCallerContext?.(created.id, { + childAgentDefaultLabels: { ui: "true" }, + allowCustomCwd: true, + enableVoiceTools: true, + }); + callerContextRegistered = true; + } catch (error) { + if (speakHandlerRegistered) { + this.unregisterVoiceSpeakHandler?.(created.id); + } + if (callerContextRegistered) { + this.unregisterVoiceCallerContext?.(created.id); + } + await this.agentManager.closeAgent(created.id).catch(() => undefined); + this.voiceAssistantAgentId = null; + throw error; + } this.sessionLogger.info( { @@ -4647,20 +4701,7 @@ export class Session { } private shouldAllowVoicePermission(request: AgentPermissionRequest): boolean { - const name = request.name.toLowerCase(); - if (name.includes("mcp") || name.includes("paseo") || name.includes("speak")) { - return true; - } - if (name === "codextool") { - const metadata = request.metadata ?? {}; - const rawQuestions = metadata.questions; - if (Array.isArray(rawQuestions)) { - const text = JSON.stringify(rawQuestions).toLowerCase(); - return text.includes("mcp") || text.includes("paseo") || text.includes("speak"); - } - return false; - } - return false; + return isVoicePermissionAllowed(request); } private async processWithVoiceAgent(userText: string): Promise { @@ -4733,10 +4774,10 @@ export class Session { /** * Process user message through LLM with streaming and tool execution */ - private async processWithLLM(enableTTS: boolean, latestUserText?: string): Promise { + private async processVoiceTurn(enableTTS: boolean, latestUserText?: string): Promise { try { if (!enableTTS) { - this.sessionLogger.warn("Ignoring non-voice processWithLLM call; voice is agent-only"); + this.sessionLogger.warn("Ignoring non-voice processVoiceTurn call; voice is agent-only"); return; } const normalized = (latestUserText ?? "").trim(); diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index 01bf369ea..9cef0f608 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "vitest"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; +import { randomUUID } from "node:crypto"; import pino from "pino"; import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; @@ -186,7 +187,7 @@ describe("speech models (download E2E)", () => { }; }); - await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`); + await ctx.client.setVoiceConversation(true, randomUUID()); for (let offset = 0; offset < pcm16.length; offset += chunkBytes) { const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); const isLast = offset + chunkBytes >= pcm16.length; diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts index 964f5b8df..921427d08 100644 --- a/packages/server/src/server/speech/speech-config-resolver.ts +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -5,14 +5,14 @@ import type { TTSConfig } from "./providers/openai/tts.js"; import type { PersistedConfig } from "../persisted-config.js"; import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/sherpa/model-catalog.js"; - -const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8"; -const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8"; +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + type SpeechProviderId, +} from "./speech-types.js"; const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech"); const DEFAULT_OPENAI_TTS_MODEL: TTSConfig["model"] = "tts-1"; -const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; - -type SpeechProviderId = "openai" | "local"; function parseSpeechProviderId(value: unknown): SpeechProviderId | null { if (typeof value !== "string") return null; diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts index 0349b8291..d06f953fa 100644 --- a/packages/server/src/server/speech/speech-runtime.ts +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -25,12 +25,12 @@ import { type SherpaOnnxModelId, } from "./providers/local/sherpa/model-catalog.js"; import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js"; - -type SpeechProviderId = "openai" | "local"; - -const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8"; -const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8"; -const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + type SpeechProviderId, +} from "./speech-types.js"; type LocalSttEngine = | { kind: "offline"; engine: SherpaOfflineRecognizerEngine } diff --git a/packages/server/src/server/speech/speech-types.ts b/packages/server/src/server/speech/speech-types.ts new file mode 100644 index 000000000..459d48eb2 --- /dev/null +++ b/packages/server/src/server/speech/speech-types.ts @@ -0,0 +1,10 @@ +import type { + LocalSttModelId, + LocalTtsModelId, +} from "./providers/local/sherpa/model-catalog.js"; + +export type SpeechProviderId = "openai" | "local"; + +export const DEFAULT_LOCAL_STT_MODEL: LocalSttModelId = "parakeet-tdt-0.6b-v3-int8"; +export const DEFAULT_LOCAL_TTS_MODEL: LocalTtsModelId = "pocket-tts-onnx-int8"; +export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; diff --git a/packages/server/src/server/voice-local-agent.e2e.test.ts b/packages/server/src/server/voice-local-agent.e2e.test.ts index 0de10e6a1..e61ab19e4 100644 --- a/packages/server/src/server/voice-local-agent.e2e.test.ts +++ b/packages/server/src/server/voice-local-agent.e2e.test.ts @@ -1,4 +1,5 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { randomUUID } from "node:crypto"; import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js"; @@ -64,7 +65,7 @@ function waitForSignal( test( "routes voice turns through local agent speak tool", async () => { - await ctx.client.setVoiceConversation(true, `voice-local-agent-${Date.now()}`); + await ctx.client.setVoiceConversation(true, randomUUID()); const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => { const offAudio = ctx.client.on("audio_output", (msg) => { diff --git a/packages/server/src/server/voice-mcp-bridge.ts b/packages/server/src/server/voice-mcp-bridge.ts index e2f32b028..ed6728fb6 100644 --- a/packages/server/src/server/voice-mcp-bridge.ts +++ b/packages/server/src/server/voice-mcp-bridge.ts @@ -221,17 +221,35 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro const socket = net.createConnection(parsed.socketPath); const stdioTransport = new StdioServerTransport(process.stdin, process.stdout); - let ready = false; let socketBuffer = ""; + let stdioStarted = false; + const pendingMcpMessages: JSONRPCMessage[] = []; + + let resolveReady: (() => void) | null = null; + let rejectReady: ((error: Error) => void) | null = null; + const readyPromise = new Promise((resolve, reject) => { + resolveReady = () => { + resolveReady = null; + rejectReady = null; + resolve(); + }; + rejectReady = (error: Error) => { + resolveReady = null; + rejectReady = null; + reject(error); + }; + }); + + const failReady = (message: string) => { + if (rejectReady) { + rejectReady(new Error(message)); + } + }; const sendEnvelope = (payload: BridgeEnvelope) => { socket.write(encodeEnvelope(payload)); }; - const closeWithError = (message: string): never => { - throw new Error(message); - }; - socket.on("data", (chunk) => { socketBuffer += chunk.toString("utf8"); while (true) { @@ -241,23 +259,41 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro socketBuffer = socketBuffer.slice(newlineIndex + 1); if (!line) continue; - const envelope = parseEnvelope(line); + let envelope: BridgeEnvelope; + try { + envelope = parseEnvelope(line); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + bridgeLogger.error({ err: error }, "Failed to parse voice MCP bridge envelope"); + failReady(`Failed to parse voice MCP bridge envelope: ${message}`); + socket.destroy(); + return; + } if (envelope.type === "ready") { - ready = true; + resolveReady?.(); continue; } if (envelope.type === "error") { + failReady(`Voice MCP bridge error: ${envelope.message}`); socket.destroy(); - closeWithError(`Voice MCP bridge error: ${envelope.message}`); + return; } if (envelope.type === "mcp") { - void stdioTransport.send(envelope.message); + if (!stdioStarted) { + pendingMcpMessages.push(envelope.message); + continue; + } + void stdioTransport.send(envelope.message).catch((error) => { + bridgeLogger.error({ err: error }, "Failed to forward MCP message to stdio transport"); + socket.destroy(); + }); } } }); socket.on("error", (error) => { bridgeLogger.error({ err: error }, "Voice MCP bridge socket client error"); + failReady(`Voice MCP bridge socket client error: ${error.message}`); }); await new Promise((resolve, reject) => { @@ -267,21 +303,13 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro sendEnvelope({ type: "init", callerAgentId: parsed.callerAgentId }); - // Wait for bridge readiness before forwarding stdio messages. - await new Promise((resolve, reject) => { - const deadline = setTimeout(() => { + const readyTimeoutPromise = new Promise((_, reject) => { + const timeout = setTimeout(() => { reject(new Error("Timed out waiting for voice MCP bridge initialization")); }, 10000); - const checkReady = () => { - if (ready) { - clearTimeout(deadline); - resolve(); - } else { - setTimeout(checkReady, 25); - } - }; - checkReady(); + readyPromise.finally(() => clearTimeout(timeout)).catch(() => undefined); }); + await Promise.race([readyPromise, readyTimeoutPromise]); stdioTransport.onmessage = (message) => { sendEnvelope({ type: "mcp", message }); @@ -295,6 +323,10 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro }; await stdioTransport.start(); + stdioStarted = true; + for (const message of pendingMcpMessages) { + await stdioTransport.send(message); + } await new Promise((resolve) => { socket.once("close", () => resolve()); diff --git a/packages/server/src/server/voice-permission-policy.test.ts b/packages/server/src/server/voice-permission-policy.test.ts new file mode 100644 index 000000000..6ca551183 --- /dev/null +++ b/packages/server/src/server/voice-permission-policy.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from "vitest"; + +import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js"; +import { isVoicePermissionAllowed } from "./voice-permission-policy.js"; + +function buildRequest(partial: Partial): AgentPermissionRequest { + return { + id: "req-1", + provider: "codex", + name: "unknown", + kind: "tool", + ...partial, + }; +} + +describe("isVoicePermissionAllowed", () => { + test("allows speak tool", () => { + const result = isVoicePermissionAllowed( + buildRequest({ name: "speak" }) + ); + expect(result).toBe(true); + }); + + test("allows explicit MCP/paseo tool names", () => { + expect( + isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" })) + ).toBe(true); + expect( + isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" })) + ).toBe(true); + }); + + test("denies non-tool permission kinds", () => { + const result = isVoicePermissionAllowed( + buildRequest({ kind: "mode", name: "mcp__paseo__create_agent" }) + ); + expect(result).toBe(false); + }); + + test("allows codextool only when metadata references MCP/paseo", () => { + const allowed = isVoicePermissionAllowed( + buildRequest({ + name: "codextool", + metadata: { + questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }], + }, + }) + ); + expect(allowed).toBe(true); + }); + + test("denies codextool when metadata includes shell-like operations", () => { + const denied = isVoicePermissionAllowed( + buildRequest({ + name: "codextool", + metadata: { + questions: [{ question: "Allow codextool to execute shell command?" }], + }, + }) + ); + expect(denied).toBe(false); + }); +}); diff --git a/packages/server/src/server/voice-permission-policy.ts b/packages/server/src/server/voice-permission-policy.ts new file mode 100644 index 000000000..0c0f77da0 --- /dev/null +++ b/packages/server/src/server/voice-permission-policy.ts @@ -0,0 +1,66 @@ +import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js"; + +const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"]; +const DENIED_TOKEN_SET = [ + "bash", + "shell", + "terminal", + "command", + "execute", + "edit", + "write", + "read", + "fetch", + "http", + "web", +]; + +function containsAny(text: string, tokens: readonly string[]): boolean { + return tokens.some((token) => text.includes(token)); +} + +function stringifyMetadata(metadata: unknown): string { + try { + return JSON.stringify(metadata ?? {})?.toLowerCase() ?? ""; + } catch { + return ""; + } +} + +/** + * Voice assistant policy: allow only MCP/paseo/speak tool requests. + * All non-tool permission requests are denied. + */ +export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean { + if (request.kind !== "tool") { + return false; + } + + const normalizedName = request.name.trim().toLowerCase(); + if (!normalizedName) { + return false; + } + + if (normalizedName === "speak") { + return true; + } + if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) { + return true; + } + + if (normalizedName !== "codextool") { + return false; + } + + const metadataText = stringifyMetadata({ + metadata: request.metadata ?? null, + input: request.input ?? null, + }); + if (!metadataText) { + return false; + } + + const mentionsAllowedTooling = containsAny(metadataText, ALLOWED_TOKEN_SET); + const mentionsDeniedTooling = containsAny(metadataText, DENIED_TOKEN_SET); + return mentionsAllowedTooling && !mentionsDeniedTooling; +} diff --git a/packages/server/src/server/voice-types.ts b/packages/server/src/server/voice-types.ts new file mode 100644 index 000000000..b5e75fa58 --- /dev/null +++ b/packages/server/src/server/voice-types.ts @@ -0,0 +1,18 @@ +export type VoiceSpeakHandler = (params: { + text: string; + callerAgentId: string; + signal?: AbortSignal; +}) => Promise; + +export type VoiceCallerContext = { + childAgentDefaultLabels?: Record; + lockedCwd?: string; + allowCustomCwd?: boolean; + enableVoiceTools?: boolean; +}; + +export type VoiceMcpStdioConfig = { + command: string; + baseArgs: string[]; + env?: Record; +}; diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 26d38a074..cc2d9a267 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -21,13 +21,13 @@ import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js"; +import type { + VoiceCallerContext, + VoiceMcpStdioConfig, + VoiceSpeakHandler, +} from "./voice-types.js"; export type AgentMcpTransportFactory = () => Promise; -type VoiceMcpStdioConfig = { - command: string; - baseArgs: string[]; - env?: Record; -}; type WebSocketServerConfig = { allowedOrigins: Set; @@ -91,17 +91,9 @@ export class VoiceAssistantWebSocketServer { } | null; private readonly voiceSpeakHandlers = new Map< string, - (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise - >(); - private readonly voiceCallerContexts = new Map< - string, - { - childAgentDefaultLabels?: Record; - lockedCwd?: string; - allowCustomCwd?: boolean; - enableVoiceTools?: boolean; - } + VoiceSpeakHandler >(); + private readonly voiceCallerContexts = new Map(); constructor( server: HTTPServer, @@ -236,23 +228,23 @@ export class VoiceAssistantWebSocketServer { const clientId = `client-${++this.clientIdCounter}`; const connectionLogger = this.logger.child({ clientId }); - const session = new Session( + const session = new Session({ clientId, - (msg) => { + onMessage: (msg) => { this.sendToClient(ws, wrapSessionMessage(msg)); }, - connectionLogger.child({ module: "session" }), - this.downloadTokenStore, - this.pushTokenStore, - this.paseoHome, - this.agentManager, - this.agentStorage, - this.createAgentMcpTransport, - this.stt, - this.tts, - this.terminalManager, - this.voice ?? undefined, - { + logger: connectionLogger.child({ module: "session" }), + downloadTokenStore: this.downloadTokenStore, + pushTokenStore: this.pushTokenStore, + paseoHome: this.paseoHome, + agentManager: this.agentManager, + agentStorage: this.agentStorage, + createAgentMcpTransport: this.createAgentMcpTransport, + stt: this.stt, + tts: this.tts, + terminalManager: this.terminalManager, + voice: this.voice ?? undefined, + voiceBridge: { registerVoiceSpeakHandler: (agentId, handler) => { this.voiceSpeakHandlers.set(agentId, handler); }, @@ -266,8 +258,8 @@ export class VoiceAssistantWebSocketServer { this.voiceCallerContexts.delete(agentId); }, }, - this.dictation ?? undefined - ); + dictation: this.dictation ?? undefined, + }); this.sessions.set(ws, session); @@ -306,18 +298,13 @@ export class VoiceAssistantWebSocketServer { public resolveVoiceSpeakHandler( callerAgentId: string - ): ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise) | null { + ): VoiceSpeakHandler | null { return this.voiceSpeakHandlers.get(callerAgentId) ?? null; } public resolveVoiceCallerContext( callerAgentId: string - ): { - childAgentDefaultLabels?: Record; - lockedCwd?: string; - allowCustomCwd?: boolean; - enableVoiceTools?: boolean; - } | null { + ): VoiceCallerContext | null { return this.voiceCallerContexts.get(callerAgentId) ?? null; }