jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

voice: harden local voice stack and enforce UUID agent ids

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-06 21:27:50 +07:00
parent a1008c3668
commit 8c70b3d86b
23 changed files with 574 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
-12
View File
@@ -4688,17 +4688,6 @@
"version": "1.1.12",
"license": "MIT"
},
"node_modules/@openrouter/ai-sdk-provider": {
"version": "1.2.0",
"license": "Apache-2.0",
"engines": {
"node": ">=18"
},
"peerDependencies": {
"ai": "^5.0.0",
"zod": "^3.24.1 || ^v4"
}
},
"node_modules/@opentelemetry/api": {
"version": "1.9.0",
"license": "Apache-2.0",
@@ -20286,7 +20275,6 @@
"@lezer/python": "^1.1.18",
"@modelcontextprotocol/sdk": "^1.20.1",
"@opencode-ai/sdk": "^1.1.12",
"@openrouter/ai-sdk-provider": "^1.2.0",
"@sctg/sentencepiece-js": "^1.1.0",
"@xterm/headless": "^6.0.0",
"ai": "^5.0.76",
packages/cli/src/cli.ts
+2 -2
View File
@@ -144,10 +144,10 @@ export function createCli(): Command {
// Internal voice MCP stdio bridge command (hidden).
program
.command('__paseo_voice_mcp_bridge')
.command('__paseo_voice_mcp_bridge', { hidden: true })
.description('Internal voice MCP bridge command')
.argument('<callerAgentId>')
.requiredOption('--socket <path>')
.requiredOption('--caller-agent-id <id>')
.action(runVoiceMcpBridgeCommand)
return program
packages/cli/src/commands/voice-mcp-bridge.ts
+2 -2
View File
@@ -2,16 +2,16 @@ import { runVoiceMcpBridgeCli } from "@getpaseo/server";
type VoiceBridgeOptions = {
socket: string;
callerAgentId: string;
};
export async function runVoiceMcpBridgeCommand(
callerAgentId: string,
options: VoiceBridgeOptions
): Promise<void> {
await runVoiceMcpBridgeCli([
"--socket",
options.socket,
"--caller-agent-id",
callerAgentId,
options.callerAgentId,
]);
}
packages/server/package.json
+2 -3
View File
@@ -36,7 +36,7 @@
"@lezer/python": "^1.1.18",
"@modelcontextprotocol/sdk": "^1.20.1",
"@opencode-ai/sdk": "^1.1.12",
"@openrouter/ai-sdk-provider": "^1.2.0",
"@sctg/sentencepiece-js": "^1.1.0",
"@xterm/headless": "^6.0.0",
"ai": "^5.0.76",
"ajv": "^8.17.1",
@@ -45,13 +45,12 @@
"express-basic-auth": "^1.2.1",
"mnemonic-id": "^3.2.7",
"node-pty": "^1.0.0",
"onnxruntime-node": "^1.23.0",
"openai": "^4.20.0",
"pino": "^10.2.0",
"pino-pretty": "^13.1.3",
"playwright": "^1.56.1",
"qrcode": "^1.5.4",
"@sctg/sentencepiece-js": "^1.1.0",
"onnxruntime-node": "^1.23.0",
"sherpa-onnx": "^1.12.23",
"sherpa-onnx-node": "^1.12.23",
"strip-ansi": "^7.1.2",
packages/server/src/server/agent/activity-curator.test.ts
+16
View File
@@ -66,6 +66,22 @@ describe("curateAgentActivity", () => {
expect(result).toBe("[ListFiles]");
});
test("does not treat generic double-underscore tool names as MCP calls", () => {
const timeline: AgentTimelineItem[] = [
{
type: "tool_call",
callId: "call-1",
name: "custom__tool",
input: {},
status: "completed",
},
];
const result = curateAgentActivity(timeline);
expect(result).toBe("[custom__tool]");
});
test("serializes todo items as [Tasks]", () => {
const timeline: AgentTimelineItem[] = [
{
packages/server/src/server/agent/activity-curator.ts
+5 -4
View File
@@ -29,10 +29,11 @@ function flushBuffers(lines: string[], buffers: { message: string; thought: stri
function isLikelyMcpToolCall(name: string): boolean {
const normalized = name.toLowerCase();
return (
normalized.includes("mcp") ||
normalized.includes("paseo") ||
normalized.includes("__") ||
normalized === "speak"
normalized === "speak" ||
normalized.startsWith("mcp") ||
normalized.includes("mcp__") ||
normalized.startsWith("paseo") ||
normalized.includes("paseo__")
);
}
packages/server/src/server/agent/agent-manager.test.ts
+91 -20
View File
@@ -118,7 +118,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "agent-without-model",
idFactory: () => "00000000-0000-4000-8000-000000000101",
});
const snapshot = await manager.createAgent({
@@ -149,7 +149,7 @@ describe("AgentManager", () => {
).rejects.toThrow("Working directory does not exist");
});
test("createAgent persists provided title before returning", async () => {
test("createAgent fails when generated agent ID is not a UUID", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
@@ -159,7 +159,52 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "agent-with-title",
idFactory: () => "not-a-uuid",
});
await expect(
manager.createAgent({
provider: "codex",
cwd: workdir,
})
).rejects.toThrow("createAgent: agentId must be a UUID");
});
test("createAgent fails when explicit agent ID is not a UUID", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
const manager = new AgentManager({
clients: {
codex: new TestAgentClient(),
},
registry: storage,
logger,
});
await expect(
manager.createAgent(
{
provider: "codex",
cwd: workdir,
},
"not-a-uuid"
)
).rejects.toThrow("createAgent: agentId must be a UUID");
});
test("createAgent persists provided title before returning", async () => {
const agentId = "00000000-0000-4000-8000-000000000102";
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
const manager = new AgentManager({
clients: {
codex: new TestAgentClient(),
},
registry: storage,
logger,
idFactory: () => agentId,
});
const snapshot = await manager.createAgent({
@@ -168,12 +213,12 @@ describe("AgentManager", () => {
title: "Fix Login Bug",
});
expect(snapshot.id).toBe("agent-with-title");
expect(snapshot.id).toBe(agentId);
expect(snapshot.lifecycle).toBe("idle");
const persisted = await storage.get("agent-with-title");
const persisted = await storage.get(agentId);
expect(persisted?.title).toBe("Fix Login Bug");
expect(persisted?.id).toBe("agent-with-title");
expect(persisted?.id).toBe(agentId);
});
test("createAgent populates runtimeInfo after session creation", async () => {
@@ -186,7 +231,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "agent-with-runtime-info",
idFactory: () => "00000000-0000-4000-8000-000000000103",
});
const snapshot = await manager.createAgent({
@@ -211,7 +256,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "agent-with-run-runtime",
idFactory: () => "00000000-0000-4000-8000-000000000104",
});
const snapshot = await manager.createAgent({
@@ -231,6 +276,10 @@ describe("AgentManager", () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
const generatedAgentIds = [
"00000000-0000-4000-8000-000000000105",
"00000000-0000-4000-8000-000000000106",
];
let agentCounter = 0;
const manager = new AgentManager({
clients: {
@@ -238,7 +287,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => `agent-${agentCounter++}`,
idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(),
});
// Create a normal agent
@@ -262,6 +311,7 @@ describe("AgentManager", () => {
});
test("getAgent returns internal agents by ID", async () => {
const internalAgentId = "00000000-0000-4000-8000-000000000107";
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
@@ -271,7 +321,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "internal-agent",
idFactory: () => internalAgentId,
});
await manager.createAgent({
@@ -281,7 +331,7 @@ describe("AgentManager", () => {
internal: true,
});
const agent = manager.getAgent("internal-agent");
const agent = manager.getAgent(internalAgentId);
expect(agent).not.toBeNull();
expect(agent?.internal).toBe(true);
});
@@ -290,6 +340,10 @@ describe("AgentManager", () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
const generatedAgentIds = [
"00000000-0000-4000-8000-000000000108",
"00000000-0000-4000-8000-000000000109",
];
let agentCounter = 0;
const manager = new AgentManager({
clients: {
@@ -297,7 +351,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => `agent-${agentCounter++}`,
idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(),
});
const receivedEvents: string[] = [];
@@ -323,11 +377,12 @@ describe("AgentManager", () => {
});
// Should only have events from the normal agent
expect(receivedEvents.filter((id) => id === "agent-0").length).toBeGreaterThan(0);
expect(receivedEvents.filter((id) => id === "agent-1").length).toBe(0);
expect(receivedEvents.filter((id) => id === generatedAgentIds[0]).length).toBeGreaterThan(0);
expect(receivedEvents.filter((id) => id === generatedAgentIds[1]).length).toBe(0);
});
test("subscribe emits state events for internal agents when subscribed by agentId", async () => {
const internalAgentId = "00000000-0000-4000-8000-000000000110";
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
@@ -337,7 +392,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "internal-agent",
idFactory: () => internalAgentId,
});
const receivedEvents: string[] = [];
@@ -348,7 +403,7 @@ describe("AgentManager", () => {
receivedEvents.push(event.agent.id);
}
},
{ agentId: "internal-agent", replayState: false }
{ agentId: internalAgentId, replayState: false }
);
await manager.createAgent({
@@ -359,10 +414,26 @@ describe("AgentManager", () => {
});
// Should receive events when subscribed by specific agentId
expect(receivedEvents.filter((id) => id === "internal-agent").length).toBeGreaterThan(0);
expect(receivedEvents.filter((id) => id === internalAgentId).length).toBeGreaterThan(0);
});
test("subscribe fails when filter agentId is not a UUID", () => {
const manager = new AgentManager({
clients: {
codex: new TestAgentClient(),
},
logger,
});
expect(() =>
manager.subscribe(() => {}, {
agentId: "invalid-agent-id",
})
).toThrow("subscribe: agentId must be a UUID");
});
test("onAgentAttention is not called for internal agents", async () => {
const internalAgentId = "00000000-0000-4000-8000-000000000111";
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
@@ -373,7 +444,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "internal-agent",
idFactory: () => internalAgentId,
onAgentAttention: ({ agentId }) => {
attentionCalls.push(agentId);
},
@@ -481,7 +552,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "plan-mode-agent",
idFactory: () => "00000000-0000-4000-8000-000000000112",
});
// Create agent in plan mode
@@ -608,7 +679,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "close-race-agent",
idFactory: () => "00000000-0000-4000-8000-000000000113",
});
const snapshot = await manager.createAgent({
packages/server/src/server/agent/agent-manager.ts
+33 -10
View File
@@ -6,6 +6,7 @@ import {
type AgentLifecycleStatus,
} from "../../shared/agent-lifecycle.js";
import type { Logger } from "pino";
import { z } from "zod";
import type {
AgentCapabilityFlags,
@@ -183,6 +184,7 @@ const BUSY_STATUSES: AgentLifecycleStatus[] = [
"initializing",
"running",
];
const AgentIdSchema = z.string().uuid();
function isAgentBusy(status: AgentLifecycleStatus): boolean {
return BUSY_STATUSES.includes(status);
@@ -202,6 +204,14 @@ function createAbortError(
return Object.assign(new Error(message), { name: "AbortError" });
}
function validateAgentId(agentId: string, source: string): string {
const result = AgentIdSchema.safeParse(agentId);
if (!result.success) {
throw new Error(`${source}: agentId must be a UUID`);
}
return result.data;
}
export class AgentManager {
private readonly clients = new Map<AgentProvider, AgentClient>();
private readonly agents = new Map<string, ActiveManagedAgent>();
@@ -239,9 +249,13 @@ export class AgentManager {
}
subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void {
const targetAgentId =
options?.agentId == null
? null
: validateAgentId(options.agentId, "subscribe");
const record: SubscriptionRecord = {
callback,
agentId: options?.agentId ?? null,
agentId: targetAgentId,
};
this.subscribers.add(record);
@@ -335,7 +349,10 @@ export class AgentManager {
options?: { labels?: Record<string, string> }
): Promise<ManagedAgent> {
// Generate agent ID early so we can use it in MCP config
const resolvedAgentId = agentId ?? this.idFactory();
const resolvedAgentId = validateAgentId(
agentId ?? this.idFactory(),
"createAgent"
);
const normalizedConfig = await this.normalizeConfig(config, {
labels: options?.labels,
agentId: resolvedAgentId,
@@ -365,6 +382,10 @@ export class AgentManager {
labels?: Record<string, string>;
}
): Promise<ManagedAgent> {
const resolvedAgentId = validateAgentId(
agentId ?? this.idFactory(),
"resumeAgent"
);
const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
const mergedConfig = {
...metadata,
@@ -381,7 +402,7 @@ export class AgentManager {
return this.registerSession(
session,
normalizedConfig,
agentId ?? this.idFactory(),
resolvedAgentId,
options
);
}
@@ -982,13 +1003,14 @@ export class AgentManager {
labels?: Record<string, string>;
}
): Promise<ManagedAgent> {
if (this.agents.has(agentId)) {
throw new Error(`Agent with id ${agentId} already exists`);
const resolvedAgentId = validateAgentId(agentId, "registerSession");
if (this.agents.has(resolvedAgentId)) {
throw new Error(`Agent with id ${resolvedAgentId} already exists`);
}
const now = new Date();
const managed = {
id: agentId,
id: resolvedAgentId,
provider: config.provider,
cwd: config.cwd,
session,
@@ -1011,9 +1033,9 @@ export class AgentManager {
labels: options?.labels ?? {},
} as ActiveManagedAgent;
this.agents.set(agentId, managed);
this.agents.set(resolvedAgentId, managed);
// Initialize previousStatus to track transitions
this.previousStatuses.set(agentId, managed.lifecycle);
this.previousStatuses.set(resolvedAgentId, managed.lifecycle);
await this.refreshRuntimeInfo(managed);
await this.persistSnapshot(managed, {
title: config.title ?? null,
@@ -1353,9 +1375,10 @@ export class AgentManager {
}
private requireAgent(id: string): ActiveManagedAgent {
const agent = this.agents.get(id);
const normalizedId = validateAgentId(id, "requireAgent");
const agent = this.agents.get(normalizedId);
if (!agent) {
throw new Error(`Unknown agent '${id}'`);
throw new Error(`Unknown agent '${normalizedId}'`);
}
return agent;
}
packages/server/src/server/agent/mcp-server.ts
+6 -7
View File
@@ -28,6 +28,10 @@ import { AgentStorage } from "./agent-storage.js";
import { createWorktree } from "../../utils/worktree.js";
import { WaitForAgentTracker } from "./wait-for-agent-tracker.js";
import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js";
import type {
VoiceCallerContext,
VoiceSpeakHandler,
} from "../voice-types.js";
export interface AgentMcpServerOptions {
agentManager: AgentManager;
@@ -44,15 +48,10 @@ export interface AgentMcpServerOptions {
*/
resolveSpeakHandler?: (
callerAgentId: string
) => ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>) | null;
) => VoiceSpeakHandler | null;
resolveCallerContext?: (
callerAgentId: string
) => {
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
} | null;
) => VoiceCallerContext | null;
enableVoiceTools?: boolean;
logger: Logger;
}
packages/server/src/server/bootstrap.smoke.test.ts
+2 -2
View File
@@ -9,9 +9,9 @@ import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js";
import { createTestAgentClients } from "./test-utils/fake-agent-client.js";
describe("paseo daemon bootstrap", () => {
test.runIf(Boolean(process.env.OPENAI_API_KEY))("starts and serves health endpoint", async () => {
test("starts and serves health endpoint", async () => {
const daemonHandle = await createTestPaseoDaemon({
openai: { apiKey: process.env.OPENAI_API_KEY! },
openai: { apiKey: "test-openai-api-key" },
speech: {
dictationSttProvider: "openai",
voiceSttProvider: "openai",
packages/server/src/server/bootstrap.ts
+12 -4
View File
@@ -73,24 +73,32 @@ type AgentMcpTransportMap = Map<string, StreamableHTTPServerTransport>;
function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } {
const explicit = process.env.PASEO_BIN_PATH?.trim();
if (explicit) {
return { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] };
const resolved = { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] };
logger.info({ source: "PASEO_BIN_PATH", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
return resolved;
}
const argv1 = process.argv[1]?.trim();
if (!argv1) {
logger.warn("Could not resolve argv[1] for voice MCP bridge; falling back to 'paseo'");
return { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] };
const resolved = { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] };
logger.info({ source: "fallback", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
return resolved;
}
const base = path.basename(argv1).toLowerCase();
if (base.includes("tsx") && process.argv[2]) {
return {
const resolved = {
command: process.execPath,
baseArgs: [argv1, process.argv[2], "__paseo_voice_mcp_bridge"],
};
logger.info({ source: "tsx", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
return resolved;
}
return { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] };
const resolved = { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] };
logger.info({ source: "argv", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
return resolved;
}
export type PaseoOpenAIConfig = {
packages/server/src/server/daemon-client.e2e.test.ts
+5 -4
View File
@@ -3,6 +3,7 @@ import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs";
import { tmpdir, homedir } from "node:os";
import path from "node:path";
import { execSync } from "node:child_process";
import { randomUUID } from "node:crypto";
import {
createDaemonTestContext,
@@ -129,7 +130,7 @@ describe("daemon client E2E", () => {
test("handles session actions", async () => {
expect(ctx.client.isConnected).toBe(true);
const voiceConversationId = `voice-${Date.now()}`;
const voiceConversationId = randomUUID();
const loadResult = await ctx.client.loadVoiceConversation(voiceConversationId);
expect(loadResult.voiceConversationId).toBe(voiceConversationId);
expect(typeof loadResult.messageCount).toBe("number");
@@ -140,7 +141,7 @@ describe("daemon client E2E", () => {
const listResult = await ctx.client.listVoiceConversations();
expect(Array.isArray(listResult.conversations)).toBe(true);
const missingId = `missing-${Date.now()}`;
const missingId = randomUUID();
const deleteResult = await ctx.client.deleteVoiceConversation(missingId);
expect(deleteResult.voiceConversationId).toBe(missingId);
expect(deleteResult.success).toBe(false);
@@ -577,7 +578,7 @@ describe("daemon client E2E", () => {
);
test(
"does not process non-voice LLM turns via OpenRouter",
"does not process non-voice turns through the voice agent path",
async () => {
await ctx.client.setVoiceConversation(false);
@@ -628,7 +629,7 @@ describe("daemon client E2E", () => {
speechTest(
"voice mode buffers audio until isLast and emits transcription_result",
async () => {
await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`);
await ctx.client.setVoiceConversation(true, randomUUID());
const transcription = waitForSignal(30_000, (resolve) => {
const unsubscribe = ctx.client.on("transcription_result", (message) => {
packages/server/src/server/session.ts
+147 -106
View File
@@ -37,20 +37,13 @@ import {
} from "./persistence-hooks.js";
import { experimental_createMCPClient } from "ai";
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
import type {
VoiceCallerContext,
VoiceMcpStdioConfig,
VoiceSpeakHandler,
} from "./voice-types.js";
export type AgentMcpTransportFactory = () => Promise<Transport>;
type VoiceSpeakHandler = (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>;
type VoiceCallerContext = {
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
};
type VoiceMcpStdioConfig = {
command: string;
baseArgs: string[];
env?: Record<string, string>;
};
import { buildProviderRegistry } from "./agent/provider-registry.js";
import { AgentManager } from "./agent/agent-manager.js";
import type { ManagedAgent } from "./agent/agent-manager.js";
@@ -73,6 +66,7 @@ import type {
} from "./agent/agent-sdk-types.js";
import { AgentStorage, type StoredAgentRecord } from "./agent/agent-storage.js";
import { isValidAgentProvider, AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
import {
listDirectoryEntries,
readExplorerFile,
@@ -188,6 +182,7 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
);
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
const VoiceConversationIdSchema = z.string().uuid();
interface AudioBufferState {
chunks: Buffer[];
@@ -196,6 +191,42 @@ interface AudioBufferState {
totalPCMBytes: number;
}
export type SessionOptions = {
clientId: string;
onMessage: (msg: SessionOutboundMessage) => void;
logger: pino.Logger;
downloadTokenStore: DownloadTokenStore;
pushTokenStore: PushTokenStore;
paseoHome: string;
agentManager: AgentManager;
agentStorage: AgentStorage;
createAgentMcpTransport: AgentMcpTransportFactory;
stt: SpeechToTextProvider | null;
tts: TextToSpeechProvider | null;
terminalManager: TerminalManager | null;
voice?: {
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
};
voiceBridge?: {
registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void;
unregisterVoiceSpeakHandler?: (agentId: string) => void;
registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void;
unregisterVoiceCallerContext?: (agentId: string) => void;
};
dictation?: {
finalTimeoutMs?: number;
stt?: SpeechToTextProvider | null;
localModels?: {
modelsDir: string;
defaultModelIds: SherpaOnnxModelId[];
};
};
};
function convertPCMToWavBuffer(
pcmBuffer: Buffer,
sampleRate: number,
@@ -353,41 +384,24 @@ export class Session {
private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
private voiceAssistantAgentId: string | null = null;
constructor(
clientId: string,
onMessage: (msg: SessionOutboundMessage) => void,
logger: pino.Logger,
downloadTokenStore: DownloadTokenStore,
pushTokenStore: PushTokenStore,
paseoHome: string,
agentManager: AgentManager,
agentStorage: AgentStorage,
createAgentMcpTransport: AgentMcpTransportFactory,
stt: SpeechToTextProvider | null,
tts: TextToSpeechProvider | null,
terminalManager: TerminalManager | null,
voice?: {
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
},
voiceBridge?: {
registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void;
unregisterVoiceSpeakHandler?: (agentId: string) => void;
registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void;
unregisterVoiceCallerContext?: (agentId: string) => void;
},
dictation?: {
finalTimeoutMs?: number;
stt?: SpeechToTextProvider | null;
localModels?: {
modelsDir: string;
defaultModelIds: SherpaOnnxModelId[];
};
}
) {
constructor(options: SessionOptions) {
const {
clientId,
onMessage,
logger,
downloadTokenStore,
pushTokenStore,
paseoHome,
agentManager,
agentStorage,
createAgentMcpTransport,
stt,
tts,
terminalManager,
voice,
voiceBridge,
dictation,
} = options;
this.clientId = clientId;
this.sessionId = uuidv4();
this.onMessage = onMessage;
@@ -1148,12 +1162,16 @@ export class Session {
voiceConversationId: string,
requestId: string
): Promise<void> {
this.voiceAssistantAgentId = voiceConversationId;
const normalizedVoiceConversationId = this.parseVoiceConversationId(
voiceConversationId,
"load_voice_conversation_request"
);
this.voiceAssistantAgentId = normalizedVoiceConversationId;
this.emit({
type: "voice_conversation_loaded",
payload: {
voiceConversationId,
voiceConversationId: normalizedVoiceConversationId,
messageCount: 0,
requestId,
},
@@ -1216,13 +1234,17 @@ export class Session {
voiceConversationId: string,
requestId: string
): Promise<void> {
const normalizedVoiceConversationId = this.parseVoiceConversationId(
voiceConversationId,
"delete_voice_conversation_request"
);
try {
const record = await this.agentStorage.get(voiceConversationId);
const record = await this.agentStorage.get(normalizedVoiceConversationId);
if (!record || record.labels?.surface !== "voice") {
this.emit({
type: "delete_voice_conversation_response",
payload: {
voiceConversationId,
voiceConversationId: normalizedVoiceConversationId,
success: false,
error: "Voice conversation not found",
requestId,
@@ -1231,32 +1253,32 @@ export class Session {
return;
}
const live = this.agentManager.getAgent(voiceConversationId);
const live = this.agentManager.getAgent(normalizedVoiceConversationId);
if (live) {
await this.agentManager.closeAgent(voiceConversationId);
await this.agentManager.closeAgent(normalizedVoiceConversationId);
}
await this.agentStorage.remove(voiceConversationId);
await this.agentStorage.remove(normalizedVoiceConversationId);
this.emit({
type: "delete_voice_conversation_response",
payload: {
voiceConversationId,
voiceConversationId: normalizedVoiceConversationId,
success: true,
requestId,
},
});
this.sessionLogger.info(
{ voiceConversationId },
`Deleted voice conversation ${voiceConversationId}`
{ voiceConversationId: normalizedVoiceConversationId },
`Deleted voice conversation ${normalizedVoiceConversationId}`
);
} catch (error: any) {
this.sessionLogger.error(
{ err: error, voiceConversationId },
`Failed to delete voice conversation ${voiceConversationId}`
{ err: error, voiceConversationId: normalizedVoiceConversationId },
`Failed to delete voice conversation ${normalizedVoiceConversationId}`
);
this.emit({
type: "delete_voice_conversation_response",
payload: {
voiceConversationId,
voiceConversationId: normalizedVoiceConversationId,
success: false,
error: error.message,
requestId,
@@ -1393,15 +1415,22 @@ export class Session {
voiceConversationId?: string
): Promise<void> {
if (enabled) {
if (!voiceConversationId || voiceConversationId.trim().length === 0) {
this.sessionLogger.warn("set_voice_conversation missing voiceConversationId; ignoring");
return;
}
const candidateVoiceConversationId =
voiceConversationId && voiceConversationId.trim().length > 0
? voiceConversationId
: uuidv4();
const normalizedVoiceConversationId = this.parseVoiceConversationId(
candidateVoiceConversationId,
"set_voice_conversation"
);
this.isVoiceMode = true;
this.voiceAssistantAgentId = voiceConversationId;
this.voiceAssistantAgentId = normalizedVoiceConversationId;
this.sessionLogger.info(
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
{
voiceAssistantAgentId: this.voiceAssistantAgentId,
generatedConversationId: !voiceConversationId,
},
"Voice conversation enabled (agent-backed)"
);
return;
@@ -1414,6 +1443,14 @@ export class Session {
);
}
private parseVoiceConversationId(rawId: string, source: string): string {
const parsed = VoiceConversationIdSchema.safeParse(rawId.trim());
if (!parsed.success) {
throw new Error(`${source}: voiceConversationId must be a UUID`);
}
return parsed.data;
}
/**
* Handle text message to agent (with optional image attachments)
*/
@@ -4259,7 +4296,7 @@ export class Session {
});
// Process through LLM (voice path is agent-backed only)
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, text);
this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, text);
await this.currentStreamPromise;
}
@@ -4528,7 +4565,7 @@ export class Session {
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
this.clearSpeechInProgress("transcription complete");
this.setPhase("llm");
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, result.text);
this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, result.text);
await this.currentStreamPromise;
this.setPhase("idle");
} catch (error: any) {
@@ -4601,30 +4638,47 @@ export class Session {
},
});
this.voiceAssistantAgentId = created.id;
let speakHandlerRegistered = false;
let callerContextRegistered = false;
this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => {
const abortSignal = signal ?? this.abortController.signal;
await this.ttsManager.generateAndWaitForPlayback(
text,
(msg) => this.emit(msg),
abortSignal,
true
);
this.emit({
type: "activity_log",
payload: {
id: uuidv4(),
timestamp: new Date(),
type: "assistant",
content: text,
},
try {
this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => {
const abortSignal = signal ?? this.abortController.signal;
await this.ttsManager.generateAndWaitForPlayback(
text,
(msg) => this.emit(msg),
abortSignal,
true
);
this.emit({
type: "activity_log",
payload: {
id: uuidv4(),
timestamp: new Date(),
type: "assistant",
content: text,
},
});
});
});
this.registerVoiceCallerContext?.(created.id, {
childAgentDefaultLabels: { ui: "true" },
allowCustomCwd: true,
enableVoiceTools: true,
});
speakHandlerRegistered = true;
this.registerVoiceCallerContext?.(created.id, {
childAgentDefaultLabels: { ui: "true" },
allowCustomCwd: true,
enableVoiceTools: true,
});
callerContextRegistered = true;
} catch (error) {
if (speakHandlerRegistered) {
this.unregisterVoiceSpeakHandler?.(created.id);
}
if (callerContextRegistered) {
this.unregisterVoiceCallerContext?.(created.id);
}
await this.agentManager.closeAgent(created.id).catch(() => undefined);
this.voiceAssistantAgentId = null;
throw error;
}
this.sessionLogger.info(
{
@@ -4647,20 +4701,7 @@ export class Session {
}
private shouldAllowVoicePermission(request: AgentPermissionRequest): boolean {
const name = request.name.toLowerCase();
if (name.includes("mcp") || name.includes("paseo") || name.includes("speak")) {
return true;
}
if (name === "codextool") {
const metadata = request.metadata ?? {};
const rawQuestions = metadata.questions;
if (Array.isArray(rawQuestions)) {
const text = JSON.stringify(rawQuestions).toLowerCase();
return text.includes("mcp") || text.includes("paseo") || text.includes("speak");
}
return false;
}
return false;
return isVoicePermissionAllowed(request);
}
private async processWithVoiceAgent(userText: string): Promise<void> {
@@ -4733,10 +4774,10 @@ export class Session {
/**
* Process user message through LLM with streaming and tool execution
*/
private async processWithLLM(enableTTS: boolean, latestUserText?: string): Promise<void> {
private async processVoiceTurn(enableTTS: boolean, latestUserText?: string): Promise<void> {
try {
if (!enableTTS) {
this.sessionLogger.warn("Ignoring non-voice processWithLLM call; voice is agent-only");
this.sessionLogger.warn("Ignoring non-voice processVoiceTurn call; voice is agent-only");
return;
}
const normalized = (latestUserText ?? "").trim();
packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts
+2 -1
View File
@@ -2,6 +2,7 @@ import { describe, expect, test } from "vitest";
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { randomUUID } from "node:crypto";
import pino from "pino";
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
@@ -186,7 +187,7 @@ describe("speech models (download E2E)", () => {
};
});
await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`);
await ctx.client.setVoiceConversation(true, randomUUID());
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
const isLast = offset + chunkBytes >= pcm16.length;
packages/server/src/server/speech/speech-config-resolver.ts
+6 -6
View File
@@ -5,14 +5,14 @@ import type { TTSConfig } from "./providers/openai/tts.js";
import type { PersistedConfig } from "../persisted-config.js";
import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/sherpa/model-catalog.js";
const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8";
const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8";
import {
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
type SpeechProviderId,
} from "./speech-types.js";
const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
const DEFAULT_OPENAI_TTS_MODEL: TTSConfig["model"] = "tts-1";
const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
type SpeechProviderId = "openai" | "local";
function parseSpeechProviderId(value: unknown): SpeechProviderId | null {
if (typeof value !== "string") return null;
packages/server/src/server/speech/speech-runtime.ts
+6 -6
View File
@@ -25,12 +25,12 @@ import {
type SherpaOnnxModelId,
} from "./providers/local/sherpa/model-catalog.js";
import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js";
type SpeechProviderId = "openai" | "local";
const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8";
const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8";
const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
import {
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
type SpeechProviderId,
} from "./speech-types.js";
type LocalSttEngine =
| { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
packages/server/src/server/speech/speech-types.ts
+10
View File
@@ -0,0 +1,10 @@
import type {
LocalSttModelId,
LocalTtsModelId,
} from "./providers/local/sherpa/model-catalog.js";
export type SpeechProviderId = "openai" | "local";
export const DEFAULT_LOCAL_STT_MODEL: LocalSttModelId = "parakeet-tdt-0.6b-v3-int8";
export const DEFAULT_LOCAL_TTS_MODEL: LocalTtsModelId = "pocket-tts-onnx-int8";
export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
packages/server/src/server/voice-local-agent.e2e.test.ts
+2 -1
View File
@@ -1,4 +1,5 @@
import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { randomUUID } from "node:crypto";
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
@@ -64,7 +65,7 @@ function waitForSignal<T>(
test(
"routes voice turns through local agent speak tool",
async () => {
await ctx.client.setVoiceConversation(true, `voice-local-agent-${Date.now()}`);
await ctx.client.setVoiceConversation(true, randomUUID());
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
const offAudio = ctx.client.on("audio_output", (msg) => {
packages/server/src/server/voice-mcp-bridge.ts
+53 -21
View File
@@ -221,17 +221,35 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro
const socket = net.createConnection(parsed.socketPath);
const stdioTransport = new StdioServerTransport(process.stdin, process.stdout);
let ready = false;
let socketBuffer = "";
let stdioStarted = false;
const pendingMcpMessages: JSONRPCMessage[] = [];
let resolveReady: (() => void) | null = null;
let rejectReady: ((error: Error) => void) | null = null;
const readyPromise = new Promise<void>((resolve, reject) => {
resolveReady = () => {
resolveReady = null;
rejectReady = null;
resolve();
};
rejectReady = (error: Error) => {
resolveReady = null;
rejectReady = null;
reject(error);
};
});
const failReady = (message: string) => {
if (rejectReady) {
rejectReady(new Error(message));
}
};
const sendEnvelope = (payload: BridgeEnvelope) => {
socket.write(encodeEnvelope(payload));
};
const closeWithError = (message: string): never => {
throw new Error(message);
};
socket.on("data", (chunk) => {
socketBuffer += chunk.toString("utf8");
while (true) {
@@ -241,23 +259,41 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro
socketBuffer = socketBuffer.slice(newlineIndex + 1);
if (!line) continue;
const envelope = parseEnvelope(line);
let envelope: BridgeEnvelope;
try {
envelope = parseEnvelope(line);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
bridgeLogger.error({ err: error }, "Failed to parse voice MCP bridge envelope");
failReady(`Failed to parse voice MCP bridge envelope: ${message}`);
socket.destroy();
return;
}
if (envelope.type === "ready") {
ready = true;
resolveReady?.();
continue;
}
if (envelope.type === "error") {
failReady(`Voice MCP bridge error: ${envelope.message}`);
socket.destroy();
closeWithError(`Voice MCP bridge error: ${envelope.message}`);
return;
}
if (envelope.type === "mcp") {
void stdioTransport.send(envelope.message);
if (!stdioStarted) {
pendingMcpMessages.push(envelope.message);
continue;
}
void stdioTransport.send(envelope.message).catch((error) => {
bridgeLogger.error({ err: error }, "Failed to forward MCP message to stdio transport");
socket.destroy();
});
}
}
});
socket.on("error", (error) => {
bridgeLogger.error({ err: error }, "Voice MCP bridge socket client error");
failReady(`Voice MCP bridge socket client error: ${error.message}`);
});
await new Promise<void>((resolve, reject) => {
@@ -267,21 +303,13 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro
sendEnvelope({ type: "init", callerAgentId: parsed.callerAgentId });
// Wait for bridge readiness before forwarding stdio messages.
await new Promise<void>((resolve, reject) => {
const deadline = setTimeout(() => {
const readyTimeoutPromise = new Promise<void>((_, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Timed out waiting for voice MCP bridge initialization"));
}, 10000);
const checkReady = () => {
if (ready) {
clearTimeout(deadline);
resolve();
} else {
setTimeout(checkReady, 25);
}
};
checkReady();
readyPromise.finally(() => clearTimeout(timeout)).catch(() => undefined);
});
await Promise.race([readyPromise, readyTimeoutPromise]);
stdioTransport.onmessage = (message) => {
sendEnvelope({ type: "mcp", message });
@@ -295,6 +323,10 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro
};
await stdioTransport.start();
stdioStarted = true;
for (const message of pendingMcpMessages) {
await stdioTransport.send(message);
}
await new Promise<void>((resolve) => {
socket.once("close", () => resolve());
packages/server/src/server/voice-permission-policy.test.ts
+63
View File
@@ -0,0 +1,63 @@
import { describe, expect, test } from "vitest";
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
function buildRequest(partial: Partial<AgentPermissionRequest>): AgentPermissionRequest {
return {
id: "req-1",
provider: "codex",
name: "unknown",
kind: "tool",
...partial,
};
}
describe("isVoicePermissionAllowed", () => {
test("allows speak tool", () => {
const result = isVoicePermissionAllowed(
buildRequest({ name: "speak" })
);
expect(result).toBe(true);
});
test("allows explicit MCP/paseo tool names", () => {
expect(
isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))
).toBe(true);
expect(
isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))
).toBe(true);
});
test("denies non-tool permission kinds", () => {
const result = isVoicePermissionAllowed(
buildRequest({ kind: "mode", name: "mcp__paseo__create_agent" })
);
expect(result).toBe(false);
});
test("allows codextool only when metadata references MCP/paseo", () => {
const allowed = isVoicePermissionAllowed(
buildRequest({
name: "codextool",
metadata: {
questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }],
},
})
);
expect(allowed).toBe(true);
});
test("denies codextool when metadata includes shell-like operations", () => {
const denied = isVoicePermissionAllowed(
buildRequest({
name: "codextool",
metadata: {
questions: [{ question: "Allow codextool to execute shell command?" }],
},
})
);
expect(denied).toBe(false);
});
});
packages/server/src/server/voice-permission-policy.ts
+66
View File
@@ -0,0 +1,66 @@
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"];
const DENIED_TOKEN_SET = [
"bash",
"shell",
"terminal",
"command",
"execute",
"edit",
"write",
"read",
"fetch",
"http",
"web",
];
function containsAny(text: string, tokens: readonly string[]): boolean {
return tokens.some((token) => text.includes(token));
}
function stringifyMetadata(metadata: unknown): string {
try {
return JSON.stringify(metadata ?? {})?.toLowerCase() ?? "";
} catch {
return "";
}
}
/**
* Voice assistant policy: allow only MCP/paseo/speak tool requests.
* All non-tool permission requests are denied.
*/
export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean {
if (request.kind !== "tool") {
return false;
}
const normalizedName = request.name.trim().toLowerCase();
if (!normalizedName) {
return false;
}
if (normalizedName === "speak") {
return true;
}
if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) {
return true;
}
if (normalizedName !== "codextool") {
return false;
}
const metadataText = stringifyMetadata({
metadata: request.metadata ?? null,
input: request.input ?? null,
});
if (!metadataText) {
return false;
}
const mentionsAllowedTooling = containsAny(metadataText, ALLOWED_TOKEN_SET);
const mentionsDeniedTooling = containsAny(metadataText, DENIED_TOKEN_SET);
return mentionsAllowedTooling && !mentionsDeniedTooling;
}
packages/server/src/server/voice-types.ts
+18
View File
@@ -0,0 +1,18 @@
export type VoiceSpeakHandler = (params: {
text: string;
callerAgentId: string;
signal?: AbortSignal;
}) => Promise<void>;
export type VoiceCallerContext = {
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
};
export type VoiceMcpStdioConfig = {
command: string;
baseArgs: string[];
env?: Record<string, string>;
};
packages/server/src/server/websocket-server.ts
+25 -38
View File
@@ -21,13 +21,13 @@ import { PushTokenStore } from "./push/token-store.js";
import { PushService } from "./push/push-service.js";
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js";
import type {
VoiceCallerContext,
VoiceMcpStdioConfig,
VoiceSpeakHandler,
} from "./voice-types.js";
export type AgentMcpTransportFactory = () => Promise<Transport>;
type VoiceMcpStdioConfig = {
command: string;
baseArgs: string[];
env?: Record<string, string>;
};
type WebSocketServerConfig = {
allowedOrigins: Set<string>;
@@ -91,17 +91,9 @@ export class VoiceAssistantWebSocketServer {
} | null;
private readonly voiceSpeakHandlers = new Map<
string,
(params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>
>();
private readonly voiceCallerContexts = new Map<
string,
{
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
}
VoiceSpeakHandler
>();
private readonly voiceCallerContexts = new Map<string, VoiceCallerContext>();
constructor(
server: HTTPServer,
@@ -236,23 +228,23 @@ export class VoiceAssistantWebSocketServer {
const clientId = `client-${++this.clientIdCounter}`;
const connectionLogger = this.logger.child({ clientId });
const session = new Session(
const session = new Session({
clientId,
(msg) => {
onMessage: (msg) => {
this.sendToClient(ws, wrapSessionMessage(msg));
},
connectionLogger.child({ module: "session" }),
this.downloadTokenStore,
this.pushTokenStore,
this.paseoHome,
this.agentManager,
this.agentStorage,
this.createAgentMcpTransport,
this.stt,
this.tts,
this.terminalManager,
this.voice ?? undefined,
{
logger: connectionLogger.child({ module: "session" }),
downloadTokenStore: this.downloadTokenStore,
pushTokenStore: this.pushTokenStore,
paseoHome: this.paseoHome,
agentManager: this.agentManager,
agentStorage: this.agentStorage,
createAgentMcpTransport: this.createAgentMcpTransport,
stt: this.stt,
tts: this.tts,
terminalManager: this.terminalManager,
voice: this.voice ?? undefined,
voiceBridge: {
registerVoiceSpeakHandler: (agentId, handler) => {
this.voiceSpeakHandlers.set(agentId, handler);
},
@@ -266,8 +258,8 @@ export class VoiceAssistantWebSocketServer {
this.voiceCallerContexts.delete(agentId);
},
},
this.dictation ?? undefined
);
dictation: this.dictation ?? undefined,
});
this.sessions.set(ws, session);
@@ -306,18 +298,13 @@ export class VoiceAssistantWebSocketServer {
public resolveVoiceSpeakHandler(
callerAgentId: string
): ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>) | null {
): VoiceSpeakHandler | null {
return this.voiceSpeakHandlers.get(callerAgentId) ?? null;
}
public resolveVoiceCallerContext(
callerAgentId: string
): {
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
} | null {
): VoiceCallerContext | null {
return this.voiceCallerContexts.get(callerAgentId) ?? null;
}