voice: harden local voice stack and enforce UUID agent ids
This commit is contained in:
Generated
-12
@@ -4688,17 +4688,6 @@
|
|||||||
"version": "1.1.12",
|
"version": "1.1.12",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/@openrouter/ai-sdk-provider": {
|
|
||||||
"version": "1.2.0",
|
|
||||||
"license": "Apache-2.0",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=18"
|
|
||||||
},
|
|
||||||
"peerDependencies": {
|
|
||||||
"ai": "^5.0.0",
|
|
||||||
"zod": "^3.24.1 || ^v4"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@opentelemetry/api": {
|
"node_modules/@opentelemetry/api": {
|
||||||
"version": "1.9.0",
|
"version": "1.9.0",
|
||||||
"license": "Apache-2.0",
|
"license": "Apache-2.0",
|
||||||
@@ -20286,7 +20275,6 @@
|
|||||||
"@lezer/python": "^1.1.18",
|
"@lezer/python": "^1.1.18",
|
||||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||||
"@opencode-ai/sdk": "^1.1.12",
|
"@opencode-ai/sdk": "^1.1.12",
|
||||||
"@openrouter/ai-sdk-provider": "^1.2.0",
|
|
||||||
"@sctg/sentencepiece-js": "^1.1.0",
|
"@sctg/sentencepiece-js": "^1.1.0",
|
||||||
"@xterm/headless": "^6.0.0",
|
"@xterm/headless": "^6.0.0",
|
||||||
"ai": "^5.0.76",
|
"ai": "^5.0.76",
|
||||||
|
|||||||
@@ -144,10 +144,10 @@ export function createCli(): Command {
|
|||||||
|
|
||||||
// Internal voice MCP stdio bridge command (hidden).
|
// Internal voice MCP stdio bridge command (hidden).
|
||||||
program
|
program
|
||||||
.command('__paseo_voice_mcp_bridge')
|
.command('__paseo_voice_mcp_bridge', { hidden: true })
|
||||||
.description('Internal voice MCP bridge command')
|
.description('Internal voice MCP bridge command')
|
||||||
.argument('<callerAgentId>')
|
|
||||||
.requiredOption('--socket <path>')
|
.requiredOption('--socket <path>')
|
||||||
|
.requiredOption('--caller-agent-id <id>')
|
||||||
.action(runVoiceMcpBridgeCommand)
|
.action(runVoiceMcpBridgeCommand)
|
||||||
|
|
||||||
return program
|
return program
|
||||||
|
|||||||
@@ -2,16 +2,16 @@ import { runVoiceMcpBridgeCli } from "@getpaseo/server";
|
|||||||
|
|
||||||
type VoiceBridgeOptions = {
|
type VoiceBridgeOptions = {
|
||||||
socket: string;
|
socket: string;
|
||||||
|
callerAgentId: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export async function runVoiceMcpBridgeCommand(
|
export async function runVoiceMcpBridgeCommand(
|
||||||
callerAgentId: string,
|
|
||||||
options: VoiceBridgeOptions
|
options: VoiceBridgeOptions
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await runVoiceMcpBridgeCli([
|
await runVoiceMcpBridgeCli([
|
||||||
"--socket",
|
"--socket",
|
||||||
options.socket,
|
options.socket,
|
||||||
"--caller-agent-id",
|
"--caller-agent-id",
|
||||||
callerAgentId,
|
options.callerAgentId,
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,7 +36,7 @@
|
|||||||
"@lezer/python": "^1.1.18",
|
"@lezer/python": "^1.1.18",
|
||||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||||
"@opencode-ai/sdk": "^1.1.12",
|
"@opencode-ai/sdk": "^1.1.12",
|
||||||
"@openrouter/ai-sdk-provider": "^1.2.0",
|
"@sctg/sentencepiece-js": "^1.1.0",
|
||||||
"@xterm/headless": "^6.0.0",
|
"@xterm/headless": "^6.0.0",
|
||||||
"ai": "^5.0.76",
|
"ai": "^5.0.76",
|
||||||
"ajv": "^8.17.1",
|
"ajv": "^8.17.1",
|
||||||
@@ -45,13 +45,12 @@
|
|||||||
"express-basic-auth": "^1.2.1",
|
"express-basic-auth": "^1.2.1",
|
||||||
"mnemonic-id": "^3.2.7",
|
"mnemonic-id": "^3.2.7",
|
||||||
"node-pty": "^1.0.0",
|
"node-pty": "^1.0.0",
|
||||||
|
"onnxruntime-node": "^1.23.0",
|
||||||
"openai": "^4.20.0",
|
"openai": "^4.20.0",
|
||||||
"pino": "^10.2.0",
|
"pino": "^10.2.0",
|
||||||
"pino-pretty": "^13.1.3",
|
"pino-pretty": "^13.1.3",
|
||||||
"playwright": "^1.56.1",
|
"playwright": "^1.56.1",
|
||||||
"qrcode": "^1.5.4",
|
"qrcode": "^1.5.4",
|
||||||
"@sctg/sentencepiece-js": "^1.1.0",
|
|
||||||
"onnxruntime-node": "^1.23.0",
|
|
||||||
"sherpa-onnx": "^1.12.23",
|
"sherpa-onnx": "^1.12.23",
|
||||||
"sherpa-onnx-node": "^1.12.23",
|
"sherpa-onnx-node": "^1.12.23",
|
||||||
"strip-ansi": "^7.1.2",
|
"strip-ansi": "^7.1.2",
|
||||||
|
|||||||
@@ -66,6 +66,22 @@ describe("curateAgentActivity", () => {
|
|||||||
expect(result).toBe("[ListFiles]");
|
expect(result).toBe("[ListFiles]");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("does not treat generic double-underscore tool names as MCP calls", () => {
|
||||||
|
const timeline: AgentTimelineItem[] = [
|
||||||
|
{
|
||||||
|
type: "tool_call",
|
||||||
|
callId: "call-1",
|
||||||
|
name: "custom__tool",
|
||||||
|
input: {},
|
||||||
|
status: "completed",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = curateAgentActivity(timeline);
|
||||||
|
|
||||||
|
expect(result).toBe("[custom__tool]");
|
||||||
|
});
|
||||||
|
|
||||||
test("serializes todo items as [Tasks]", () => {
|
test("serializes todo items as [Tasks]", () => {
|
||||||
const timeline: AgentTimelineItem[] = [
|
const timeline: AgentTimelineItem[] = [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -29,10 +29,11 @@ function flushBuffers(lines: string[], buffers: { message: string; thought: stri
|
|||||||
function isLikelyMcpToolCall(name: string): boolean {
|
function isLikelyMcpToolCall(name: string): boolean {
|
||||||
const normalized = name.toLowerCase();
|
const normalized = name.toLowerCase();
|
||||||
return (
|
return (
|
||||||
normalized.includes("mcp") ||
|
normalized === "speak" ||
|
||||||
normalized.includes("paseo") ||
|
normalized.startsWith("mcp") ||
|
||||||
normalized.includes("__") ||
|
normalized.includes("mcp__") ||
|
||||||
normalized === "speak"
|
normalized.startsWith("paseo") ||
|
||||||
|
normalized.includes("paseo__")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -118,7 +118,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => "agent-without-model",
|
idFactory: () => "00000000-0000-4000-8000-000000000101",
|
||||||
});
|
});
|
||||||
|
|
||||||
const snapshot = await manager.createAgent({
|
const snapshot = await manager.createAgent({
|
||||||
@@ -149,7 +149,7 @@ describe("AgentManager", () => {
|
|||||||
).rejects.toThrow("Working directory does not exist");
|
).rejects.toThrow("Working directory does not exist");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("createAgent persists provided title before returning", async () => {
|
test("createAgent fails when generated agent ID is not a UUID", async () => {
|
||||||
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
||||||
const storagePath = join(workdir, "agents");
|
const storagePath = join(workdir, "agents");
|
||||||
const storage = new AgentStorage(storagePath, logger);
|
const storage = new AgentStorage(storagePath, logger);
|
||||||
@@ -159,7 +159,52 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => "agent-with-title",
|
idFactory: () => "not-a-uuid",
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
manager.createAgent({
|
||||||
|
provider: "codex",
|
||||||
|
cwd: workdir,
|
||||||
|
})
|
||||||
|
).rejects.toThrow("createAgent: agentId must be a UUID");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("createAgent fails when explicit agent ID is not a UUID", async () => {
|
||||||
|
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
||||||
|
const storagePath = join(workdir, "agents");
|
||||||
|
const storage = new AgentStorage(storagePath, logger);
|
||||||
|
const manager = new AgentManager({
|
||||||
|
clients: {
|
||||||
|
codex: new TestAgentClient(),
|
||||||
|
},
|
||||||
|
registry: storage,
|
||||||
|
logger,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
manager.createAgent(
|
||||||
|
{
|
||||||
|
provider: "codex",
|
||||||
|
cwd: workdir,
|
||||||
|
},
|
||||||
|
"not-a-uuid"
|
||||||
|
)
|
||||||
|
).rejects.toThrow("createAgent: agentId must be a UUID");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("createAgent persists provided title before returning", async () => {
|
||||||
|
const agentId = "00000000-0000-4000-8000-000000000102";
|
||||||
|
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
||||||
|
const storagePath = join(workdir, "agents");
|
||||||
|
const storage = new AgentStorage(storagePath, logger);
|
||||||
|
const manager = new AgentManager({
|
||||||
|
clients: {
|
||||||
|
codex: new TestAgentClient(),
|
||||||
|
},
|
||||||
|
registry: storage,
|
||||||
|
logger,
|
||||||
|
idFactory: () => agentId,
|
||||||
});
|
});
|
||||||
|
|
||||||
const snapshot = await manager.createAgent({
|
const snapshot = await manager.createAgent({
|
||||||
@@ -168,12 +213,12 @@ describe("AgentManager", () => {
|
|||||||
title: "Fix Login Bug",
|
title: "Fix Login Bug",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(snapshot.id).toBe("agent-with-title");
|
expect(snapshot.id).toBe(agentId);
|
||||||
expect(snapshot.lifecycle).toBe("idle");
|
expect(snapshot.lifecycle).toBe("idle");
|
||||||
|
|
||||||
const persisted = await storage.get("agent-with-title");
|
const persisted = await storage.get(agentId);
|
||||||
expect(persisted?.title).toBe("Fix Login Bug");
|
expect(persisted?.title).toBe("Fix Login Bug");
|
||||||
expect(persisted?.id).toBe("agent-with-title");
|
expect(persisted?.id).toBe(agentId);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("createAgent populates runtimeInfo after session creation", async () => {
|
test("createAgent populates runtimeInfo after session creation", async () => {
|
||||||
@@ -186,7 +231,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => "agent-with-runtime-info",
|
idFactory: () => "00000000-0000-4000-8000-000000000103",
|
||||||
});
|
});
|
||||||
|
|
||||||
const snapshot = await manager.createAgent({
|
const snapshot = await manager.createAgent({
|
||||||
@@ -211,7 +256,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => "agent-with-run-runtime",
|
idFactory: () => "00000000-0000-4000-8000-000000000104",
|
||||||
});
|
});
|
||||||
|
|
||||||
const snapshot = await manager.createAgent({
|
const snapshot = await manager.createAgent({
|
||||||
@@ -231,6 +276,10 @@ describe("AgentManager", () => {
|
|||||||
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
||||||
const storagePath = join(workdir, "agents");
|
const storagePath = join(workdir, "agents");
|
||||||
const storage = new AgentStorage(storagePath, logger);
|
const storage = new AgentStorage(storagePath, logger);
|
||||||
|
const generatedAgentIds = [
|
||||||
|
"00000000-0000-4000-8000-000000000105",
|
||||||
|
"00000000-0000-4000-8000-000000000106",
|
||||||
|
];
|
||||||
let agentCounter = 0;
|
let agentCounter = 0;
|
||||||
const manager = new AgentManager({
|
const manager = new AgentManager({
|
||||||
clients: {
|
clients: {
|
||||||
@@ -238,7 +287,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => `agent-${agentCounter++}`,
|
idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(),
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create a normal agent
|
// Create a normal agent
|
||||||
@@ -262,6 +311,7 @@ describe("AgentManager", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("getAgent returns internal agents by ID", async () => {
|
test("getAgent returns internal agents by ID", async () => {
|
||||||
|
const internalAgentId = "00000000-0000-4000-8000-000000000107";
|
||||||
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
||||||
const storagePath = join(workdir, "agents");
|
const storagePath = join(workdir, "agents");
|
||||||
const storage = new AgentStorage(storagePath, logger);
|
const storage = new AgentStorage(storagePath, logger);
|
||||||
@@ -271,7 +321,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => "internal-agent",
|
idFactory: () => internalAgentId,
|
||||||
});
|
});
|
||||||
|
|
||||||
await manager.createAgent({
|
await manager.createAgent({
|
||||||
@@ -281,7 +331,7 @@ describe("AgentManager", () => {
|
|||||||
internal: true,
|
internal: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
const agent = manager.getAgent("internal-agent");
|
const agent = manager.getAgent(internalAgentId);
|
||||||
expect(agent).not.toBeNull();
|
expect(agent).not.toBeNull();
|
||||||
expect(agent?.internal).toBe(true);
|
expect(agent?.internal).toBe(true);
|
||||||
});
|
});
|
||||||
@@ -290,6 +340,10 @@ describe("AgentManager", () => {
|
|||||||
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
||||||
const storagePath = join(workdir, "agents");
|
const storagePath = join(workdir, "agents");
|
||||||
const storage = new AgentStorage(storagePath, logger);
|
const storage = new AgentStorage(storagePath, logger);
|
||||||
|
const generatedAgentIds = [
|
||||||
|
"00000000-0000-4000-8000-000000000108",
|
||||||
|
"00000000-0000-4000-8000-000000000109",
|
||||||
|
];
|
||||||
let agentCounter = 0;
|
let agentCounter = 0;
|
||||||
const manager = new AgentManager({
|
const manager = new AgentManager({
|
||||||
clients: {
|
clients: {
|
||||||
@@ -297,7 +351,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => `agent-${agentCounter++}`,
|
idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(),
|
||||||
});
|
});
|
||||||
|
|
||||||
const receivedEvents: string[] = [];
|
const receivedEvents: string[] = [];
|
||||||
@@ -323,11 +377,12 @@ describe("AgentManager", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Should only have events from the normal agent
|
// Should only have events from the normal agent
|
||||||
expect(receivedEvents.filter((id) => id === "agent-0").length).toBeGreaterThan(0);
|
expect(receivedEvents.filter((id) => id === generatedAgentIds[0]).length).toBeGreaterThan(0);
|
||||||
expect(receivedEvents.filter((id) => id === "agent-1").length).toBe(0);
|
expect(receivedEvents.filter((id) => id === generatedAgentIds[1]).length).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("subscribe emits state events for internal agents when subscribed by agentId", async () => {
|
test("subscribe emits state events for internal agents when subscribed by agentId", async () => {
|
||||||
|
const internalAgentId = "00000000-0000-4000-8000-000000000110";
|
||||||
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
||||||
const storagePath = join(workdir, "agents");
|
const storagePath = join(workdir, "agents");
|
||||||
const storage = new AgentStorage(storagePath, logger);
|
const storage = new AgentStorage(storagePath, logger);
|
||||||
@@ -337,7 +392,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => "internal-agent",
|
idFactory: () => internalAgentId,
|
||||||
});
|
});
|
||||||
|
|
||||||
const receivedEvents: string[] = [];
|
const receivedEvents: string[] = [];
|
||||||
@@ -348,7 +403,7 @@ describe("AgentManager", () => {
|
|||||||
receivedEvents.push(event.agent.id);
|
receivedEvents.push(event.agent.id);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{ agentId: "internal-agent", replayState: false }
|
{ agentId: internalAgentId, replayState: false }
|
||||||
);
|
);
|
||||||
|
|
||||||
await manager.createAgent({
|
await manager.createAgent({
|
||||||
@@ -359,10 +414,26 @@ describe("AgentManager", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Should receive events when subscribed by specific agentId
|
// Should receive events when subscribed by specific agentId
|
||||||
expect(receivedEvents.filter((id) => id === "internal-agent").length).toBeGreaterThan(0);
|
expect(receivedEvents.filter((id) => id === internalAgentId).length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("subscribe fails when filter agentId is not a UUID", () => {
|
||||||
|
const manager = new AgentManager({
|
||||||
|
clients: {
|
||||||
|
codex: new TestAgentClient(),
|
||||||
|
},
|
||||||
|
logger,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(() =>
|
||||||
|
manager.subscribe(() => {}, {
|
||||||
|
agentId: "invalid-agent-id",
|
||||||
|
})
|
||||||
|
).toThrow("subscribe: agentId must be a UUID");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("onAgentAttention is not called for internal agents", async () => {
|
test("onAgentAttention is not called for internal agents", async () => {
|
||||||
|
const internalAgentId = "00000000-0000-4000-8000-000000000111";
|
||||||
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
|
||||||
const storagePath = join(workdir, "agents");
|
const storagePath = join(workdir, "agents");
|
||||||
const storage = new AgentStorage(storagePath, logger);
|
const storage = new AgentStorage(storagePath, logger);
|
||||||
@@ -373,7 +444,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => "internal-agent",
|
idFactory: () => internalAgentId,
|
||||||
onAgentAttention: ({ agentId }) => {
|
onAgentAttention: ({ agentId }) => {
|
||||||
attentionCalls.push(agentId);
|
attentionCalls.push(agentId);
|
||||||
},
|
},
|
||||||
@@ -481,7 +552,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => "plan-mode-agent",
|
idFactory: () => "00000000-0000-4000-8000-000000000112",
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create agent in plan mode
|
// Create agent in plan mode
|
||||||
@@ -608,7 +679,7 @@ describe("AgentManager", () => {
|
|||||||
},
|
},
|
||||||
registry: storage,
|
registry: storage,
|
||||||
logger,
|
logger,
|
||||||
idFactory: () => "close-race-agent",
|
idFactory: () => "00000000-0000-4000-8000-000000000113",
|
||||||
});
|
});
|
||||||
|
|
||||||
const snapshot = await manager.createAgent({
|
const snapshot = await manager.createAgent({
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
type AgentLifecycleStatus,
|
type AgentLifecycleStatus,
|
||||||
} from "../../shared/agent-lifecycle.js";
|
} from "../../shared/agent-lifecycle.js";
|
||||||
import type { Logger } from "pino";
|
import type { Logger } from "pino";
|
||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
import type {
|
import type {
|
||||||
AgentCapabilityFlags,
|
AgentCapabilityFlags,
|
||||||
@@ -183,6 +184,7 @@ const BUSY_STATUSES: AgentLifecycleStatus[] = [
|
|||||||
"initializing",
|
"initializing",
|
||||||
"running",
|
"running",
|
||||||
];
|
];
|
||||||
|
const AgentIdSchema = z.string().uuid();
|
||||||
|
|
||||||
function isAgentBusy(status: AgentLifecycleStatus): boolean {
|
function isAgentBusy(status: AgentLifecycleStatus): boolean {
|
||||||
return BUSY_STATUSES.includes(status);
|
return BUSY_STATUSES.includes(status);
|
||||||
@@ -202,6 +204,14 @@ function createAbortError(
|
|||||||
return Object.assign(new Error(message), { name: "AbortError" });
|
return Object.assign(new Error(message), { name: "AbortError" });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validateAgentId(agentId: string, source: string): string {
|
||||||
|
const result = AgentIdSchema.safeParse(agentId);
|
||||||
|
if (!result.success) {
|
||||||
|
throw new Error(`${source}: agentId must be a UUID`);
|
||||||
|
}
|
||||||
|
return result.data;
|
||||||
|
}
|
||||||
|
|
||||||
export class AgentManager {
|
export class AgentManager {
|
||||||
private readonly clients = new Map<AgentProvider, AgentClient>();
|
private readonly clients = new Map<AgentProvider, AgentClient>();
|
||||||
private readonly agents = new Map<string, ActiveManagedAgent>();
|
private readonly agents = new Map<string, ActiveManagedAgent>();
|
||||||
@@ -239,9 +249,13 @@ export class AgentManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void {
|
subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void {
|
||||||
|
const targetAgentId =
|
||||||
|
options?.agentId == null
|
||||||
|
? null
|
||||||
|
: validateAgentId(options.agentId, "subscribe");
|
||||||
const record: SubscriptionRecord = {
|
const record: SubscriptionRecord = {
|
||||||
callback,
|
callback,
|
||||||
agentId: options?.agentId ?? null,
|
agentId: targetAgentId,
|
||||||
};
|
};
|
||||||
this.subscribers.add(record);
|
this.subscribers.add(record);
|
||||||
|
|
||||||
@@ -335,7 +349,10 @@ export class AgentManager {
|
|||||||
options?: { labels?: Record<string, string> }
|
options?: { labels?: Record<string, string> }
|
||||||
): Promise<ManagedAgent> {
|
): Promise<ManagedAgent> {
|
||||||
// Generate agent ID early so we can use it in MCP config
|
// Generate agent ID early so we can use it in MCP config
|
||||||
const resolvedAgentId = agentId ?? this.idFactory();
|
const resolvedAgentId = validateAgentId(
|
||||||
|
agentId ?? this.idFactory(),
|
||||||
|
"createAgent"
|
||||||
|
);
|
||||||
const normalizedConfig = await this.normalizeConfig(config, {
|
const normalizedConfig = await this.normalizeConfig(config, {
|
||||||
labels: options?.labels,
|
labels: options?.labels,
|
||||||
agentId: resolvedAgentId,
|
agentId: resolvedAgentId,
|
||||||
@@ -365,6 +382,10 @@ export class AgentManager {
|
|||||||
labels?: Record<string, string>;
|
labels?: Record<string, string>;
|
||||||
}
|
}
|
||||||
): Promise<ManagedAgent> {
|
): Promise<ManagedAgent> {
|
||||||
|
const resolvedAgentId = validateAgentId(
|
||||||
|
agentId ?? this.idFactory(),
|
||||||
|
"resumeAgent"
|
||||||
|
);
|
||||||
const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
|
const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
|
||||||
const mergedConfig = {
|
const mergedConfig = {
|
||||||
...metadata,
|
...metadata,
|
||||||
@@ -381,7 +402,7 @@ export class AgentManager {
|
|||||||
return this.registerSession(
|
return this.registerSession(
|
||||||
session,
|
session,
|
||||||
normalizedConfig,
|
normalizedConfig,
|
||||||
agentId ?? this.idFactory(),
|
resolvedAgentId,
|
||||||
options
|
options
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -982,13 +1003,14 @@ export class AgentManager {
|
|||||||
labels?: Record<string, string>;
|
labels?: Record<string, string>;
|
||||||
}
|
}
|
||||||
): Promise<ManagedAgent> {
|
): Promise<ManagedAgent> {
|
||||||
if (this.agents.has(agentId)) {
|
const resolvedAgentId = validateAgentId(agentId, "registerSession");
|
||||||
throw new Error(`Agent with id ${agentId} already exists`);
|
if (this.agents.has(resolvedAgentId)) {
|
||||||
|
throw new Error(`Agent with id ${resolvedAgentId} already exists`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const now = new Date();
|
const now = new Date();
|
||||||
const managed = {
|
const managed = {
|
||||||
id: agentId,
|
id: resolvedAgentId,
|
||||||
provider: config.provider,
|
provider: config.provider,
|
||||||
cwd: config.cwd,
|
cwd: config.cwd,
|
||||||
session,
|
session,
|
||||||
@@ -1011,9 +1033,9 @@ export class AgentManager {
|
|||||||
labels: options?.labels ?? {},
|
labels: options?.labels ?? {},
|
||||||
} as ActiveManagedAgent;
|
} as ActiveManagedAgent;
|
||||||
|
|
||||||
this.agents.set(agentId, managed);
|
this.agents.set(resolvedAgentId, managed);
|
||||||
// Initialize previousStatus to track transitions
|
// Initialize previousStatus to track transitions
|
||||||
this.previousStatuses.set(agentId, managed.lifecycle);
|
this.previousStatuses.set(resolvedAgentId, managed.lifecycle);
|
||||||
await this.refreshRuntimeInfo(managed);
|
await this.refreshRuntimeInfo(managed);
|
||||||
await this.persistSnapshot(managed, {
|
await this.persistSnapshot(managed, {
|
||||||
title: config.title ?? null,
|
title: config.title ?? null,
|
||||||
@@ -1353,9 +1375,10 @@ export class AgentManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private requireAgent(id: string): ActiveManagedAgent {
|
private requireAgent(id: string): ActiveManagedAgent {
|
||||||
const agent = this.agents.get(id);
|
const normalizedId = validateAgentId(id, "requireAgent");
|
||||||
|
const agent = this.agents.get(normalizedId);
|
||||||
if (!agent) {
|
if (!agent) {
|
||||||
throw new Error(`Unknown agent '${id}'`);
|
throw new Error(`Unknown agent '${normalizedId}'`);
|
||||||
}
|
}
|
||||||
return agent;
|
return agent;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,10 @@ import { AgentStorage } from "./agent-storage.js";
|
|||||||
import { createWorktree } from "../../utils/worktree.js";
|
import { createWorktree } from "../../utils/worktree.js";
|
||||||
import { WaitForAgentTracker } from "./wait-for-agent-tracker.js";
|
import { WaitForAgentTracker } from "./wait-for-agent-tracker.js";
|
||||||
import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js";
|
import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js";
|
||||||
|
import type {
|
||||||
|
VoiceCallerContext,
|
||||||
|
VoiceSpeakHandler,
|
||||||
|
} from "../voice-types.js";
|
||||||
|
|
||||||
export interface AgentMcpServerOptions {
|
export interface AgentMcpServerOptions {
|
||||||
agentManager: AgentManager;
|
agentManager: AgentManager;
|
||||||
@@ -44,15 +48,10 @@ export interface AgentMcpServerOptions {
|
|||||||
*/
|
*/
|
||||||
resolveSpeakHandler?: (
|
resolveSpeakHandler?: (
|
||||||
callerAgentId: string
|
callerAgentId: string
|
||||||
) => ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>) | null;
|
) => VoiceSpeakHandler | null;
|
||||||
resolveCallerContext?: (
|
resolveCallerContext?: (
|
||||||
callerAgentId: string
|
callerAgentId: string
|
||||||
) => {
|
) => VoiceCallerContext | null;
|
||||||
childAgentDefaultLabels?: Record<string, string>;
|
|
||||||
lockedCwd?: string;
|
|
||||||
allowCustomCwd?: boolean;
|
|
||||||
enableVoiceTools?: boolean;
|
|
||||||
} | null;
|
|
||||||
enableVoiceTools?: boolean;
|
enableVoiceTools?: boolean;
|
||||||
logger: Logger;
|
logger: Logger;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,9 +9,9 @@ import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js";
|
|||||||
import { createTestAgentClients } from "./test-utils/fake-agent-client.js";
|
import { createTestAgentClients } from "./test-utils/fake-agent-client.js";
|
||||||
|
|
||||||
describe("paseo daemon bootstrap", () => {
|
describe("paseo daemon bootstrap", () => {
|
||||||
test.runIf(Boolean(process.env.OPENAI_API_KEY))("starts and serves health endpoint", async () => {
|
test("starts and serves health endpoint", async () => {
|
||||||
const daemonHandle = await createTestPaseoDaemon({
|
const daemonHandle = await createTestPaseoDaemon({
|
||||||
openai: { apiKey: process.env.OPENAI_API_KEY! },
|
openai: { apiKey: "test-openai-api-key" },
|
||||||
speech: {
|
speech: {
|
||||||
dictationSttProvider: "openai",
|
dictationSttProvider: "openai",
|
||||||
voiceSttProvider: "openai",
|
voiceSttProvider: "openai",
|
||||||
|
|||||||
@@ -73,24 +73,32 @@ type AgentMcpTransportMap = Map<string, StreamableHTTPServerTransport>;
|
|||||||
function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } {
|
function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } {
|
||||||
const explicit = process.env.PASEO_BIN_PATH?.trim();
|
const explicit = process.env.PASEO_BIN_PATH?.trim();
|
||||||
if (explicit) {
|
if (explicit) {
|
||||||
return { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] };
|
const resolved = { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] };
|
||||||
|
logger.info({ source: "PASEO_BIN_PATH", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
|
||||||
|
return resolved;
|
||||||
}
|
}
|
||||||
|
|
||||||
const argv1 = process.argv[1]?.trim();
|
const argv1 = process.argv[1]?.trim();
|
||||||
if (!argv1) {
|
if (!argv1) {
|
||||||
logger.warn("Could not resolve argv[1] for voice MCP bridge; falling back to 'paseo'");
|
logger.warn("Could not resolve argv[1] for voice MCP bridge; falling back to 'paseo'");
|
||||||
return { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] };
|
const resolved = { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] };
|
||||||
|
logger.info({ source: "fallback", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
|
||||||
|
return resolved;
|
||||||
}
|
}
|
||||||
|
|
||||||
const base = path.basename(argv1).toLowerCase();
|
const base = path.basename(argv1).toLowerCase();
|
||||||
if (base.includes("tsx") && process.argv[2]) {
|
if (base.includes("tsx") && process.argv[2]) {
|
||||||
return {
|
const resolved = {
|
||||||
command: process.execPath,
|
command: process.execPath,
|
||||||
baseArgs: [argv1, process.argv[2], "__paseo_voice_mcp_bridge"],
|
baseArgs: [argv1, process.argv[2], "__paseo_voice_mcp_bridge"],
|
||||||
};
|
};
|
||||||
|
logger.info({ source: "tsx", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
|
||||||
|
return resolved;
|
||||||
}
|
}
|
||||||
|
|
||||||
return { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] };
|
const resolved = { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] };
|
||||||
|
logger.info({ source: "argv", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
|
||||||
|
return resolved;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type PaseoOpenAIConfig = {
|
export type PaseoOpenAIConfig = {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs";
|
|||||||
import { tmpdir, homedir } from "node:os";
|
import { tmpdir, homedir } from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { execSync } from "node:child_process";
|
import { execSync } from "node:child_process";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
createDaemonTestContext,
|
createDaemonTestContext,
|
||||||
@@ -129,7 +130,7 @@ describe("daemon client E2E", () => {
|
|||||||
test("handles session actions", async () => {
|
test("handles session actions", async () => {
|
||||||
expect(ctx.client.isConnected).toBe(true);
|
expect(ctx.client.isConnected).toBe(true);
|
||||||
|
|
||||||
const voiceConversationId = `voice-${Date.now()}`;
|
const voiceConversationId = randomUUID();
|
||||||
const loadResult = await ctx.client.loadVoiceConversation(voiceConversationId);
|
const loadResult = await ctx.client.loadVoiceConversation(voiceConversationId);
|
||||||
expect(loadResult.voiceConversationId).toBe(voiceConversationId);
|
expect(loadResult.voiceConversationId).toBe(voiceConversationId);
|
||||||
expect(typeof loadResult.messageCount).toBe("number");
|
expect(typeof loadResult.messageCount).toBe("number");
|
||||||
@@ -140,7 +141,7 @@ describe("daemon client E2E", () => {
|
|||||||
const listResult = await ctx.client.listVoiceConversations();
|
const listResult = await ctx.client.listVoiceConversations();
|
||||||
expect(Array.isArray(listResult.conversations)).toBe(true);
|
expect(Array.isArray(listResult.conversations)).toBe(true);
|
||||||
|
|
||||||
const missingId = `missing-${Date.now()}`;
|
const missingId = randomUUID();
|
||||||
const deleteResult = await ctx.client.deleteVoiceConversation(missingId);
|
const deleteResult = await ctx.client.deleteVoiceConversation(missingId);
|
||||||
expect(deleteResult.voiceConversationId).toBe(missingId);
|
expect(deleteResult.voiceConversationId).toBe(missingId);
|
||||||
expect(deleteResult.success).toBe(false);
|
expect(deleteResult.success).toBe(false);
|
||||||
@@ -577,7 +578,7 @@ describe("daemon client E2E", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
test(
|
test(
|
||||||
"does not process non-voice LLM turns via OpenRouter",
|
"does not process non-voice turns through the voice agent path",
|
||||||
async () => {
|
async () => {
|
||||||
await ctx.client.setVoiceConversation(false);
|
await ctx.client.setVoiceConversation(false);
|
||||||
|
|
||||||
@@ -628,7 +629,7 @@ describe("daemon client E2E", () => {
|
|||||||
speechTest(
|
speechTest(
|
||||||
"voice mode buffers audio until isLast and emits transcription_result",
|
"voice mode buffers audio until isLast and emits transcription_result",
|
||||||
async () => {
|
async () => {
|
||||||
await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`);
|
await ctx.client.setVoiceConversation(true, randomUUID());
|
||||||
|
|
||||||
const transcription = waitForSignal(30_000, (resolve) => {
|
const transcription = waitForSignal(30_000, (resolve) => {
|
||||||
const unsubscribe = ctx.client.on("transcription_result", (message) => {
|
const unsubscribe = ctx.client.on("transcription_result", (message) => {
|
||||||
|
|||||||
@@ -37,20 +37,13 @@ import {
|
|||||||
} from "./persistence-hooks.js";
|
} from "./persistence-hooks.js";
|
||||||
import { experimental_createMCPClient } from "ai";
|
import { experimental_createMCPClient } from "ai";
|
||||||
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
|
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
|
||||||
|
import type {
|
||||||
|
VoiceCallerContext,
|
||||||
|
VoiceMcpStdioConfig,
|
||||||
|
VoiceSpeakHandler,
|
||||||
|
} from "./voice-types.js";
|
||||||
|
|
||||||
export type AgentMcpTransportFactory = () => Promise<Transport>;
|
export type AgentMcpTransportFactory = () => Promise<Transport>;
|
||||||
type VoiceSpeakHandler = (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>;
|
|
||||||
type VoiceCallerContext = {
|
|
||||||
childAgentDefaultLabels?: Record<string, string>;
|
|
||||||
lockedCwd?: string;
|
|
||||||
allowCustomCwd?: boolean;
|
|
||||||
enableVoiceTools?: boolean;
|
|
||||||
};
|
|
||||||
type VoiceMcpStdioConfig = {
|
|
||||||
command: string;
|
|
||||||
baseArgs: string[];
|
|
||||||
env?: Record<string, string>;
|
|
||||||
};
|
|
||||||
import { buildProviderRegistry } from "./agent/provider-registry.js";
|
import { buildProviderRegistry } from "./agent/provider-registry.js";
|
||||||
import { AgentManager } from "./agent/agent-manager.js";
|
import { AgentManager } from "./agent/agent-manager.js";
|
||||||
import type { ManagedAgent } from "./agent/agent-manager.js";
|
import type { ManagedAgent } from "./agent/agent-manager.js";
|
||||||
@@ -73,6 +66,7 @@ import type {
|
|||||||
} from "./agent/agent-sdk-types.js";
|
} from "./agent/agent-sdk-types.js";
|
||||||
import { AgentStorage, type StoredAgentRecord } from "./agent/agent-storage.js";
|
import { AgentStorage, type StoredAgentRecord } from "./agent/agent-storage.js";
|
||||||
import { isValidAgentProvider, AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
|
import { isValidAgentProvider, AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
|
||||||
|
import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
|
||||||
import {
|
import {
|
||||||
listDirectoryEntries,
|
listDirectoryEntries,
|
||||||
readExplorerFile,
|
readExplorerFile,
|
||||||
@@ -188,6 +182,7 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(
|
|||||||
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
|
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
|
||||||
);
|
);
|
||||||
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
|
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
|
||||||
|
const VoiceConversationIdSchema = z.string().uuid();
|
||||||
|
|
||||||
interface AudioBufferState {
|
interface AudioBufferState {
|
||||||
chunks: Buffer[];
|
chunks: Buffer[];
|
||||||
@@ -196,6 +191,42 @@ interface AudioBufferState {
|
|||||||
totalPCMBytes: number;
|
totalPCMBytes: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type SessionOptions = {
|
||||||
|
clientId: string;
|
||||||
|
onMessage: (msg: SessionOutboundMessage) => void;
|
||||||
|
logger: pino.Logger;
|
||||||
|
downloadTokenStore: DownloadTokenStore;
|
||||||
|
pushTokenStore: PushTokenStore;
|
||||||
|
paseoHome: string;
|
||||||
|
agentManager: AgentManager;
|
||||||
|
agentStorage: AgentStorage;
|
||||||
|
createAgentMcpTransport: AgentMcpTransportFactory;
|
||||||
|
stt: SpeechToTextProvider | null;
|
||||||
|
tts: TextToSpeechProvider | null;
|
||||||
|
terminalManager: TerminalManager | null;
|
||||||
|
voice?: {
|
||||||
|
voiceLlmProvider?: AgentProvider | null;
|
||||||
|
voiceLlmModeId?: string | null;
|
||||||
|
voiceLlmProviderExplicit?: boolean;
|
||||||
|
voiceLlmModel?: string | null;
|
||||||
|
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
|
||||||
|
};
|
||||||
|
voiceBridge?: {
|
||||||
|
registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void;
|
||||||
|
unregisterVoiceSpeakHandler?: (agentId: string) => void;
|
||||||
|
registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void;
|
||||||
|
unregisterVoiceCallerContext?: (agentId: string) => void;
|
||||||
|
};
|
||||||
|
dictation?: {
|
||||||
|
finalTimeoutMs?: number;
|
||||||
|
stt?: SpeechToTextProvider | null;
|
||||||
|
localModels?: {
|
||||||
|
modelsDir: string;
|
||||||
|
defaultModelIds: SherpaOnnxModelId[];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
function convertPCMToWavBuffer(
|
function convertPCMToWavBuffer(
|
||||||
pcmBuffer: Buffer,
|
pcmBuffer: Buffer,
|
||||||
sampleRate: number,
|
sampleRate: number,
|
||||||
@@ -353,41 +384,24 @@ export class Session {
|
|||||||
private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
|
private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
|
||||||
private voiceAssistantAgentId: string | null = null;
|
private voiceAssistantAgentId: string | null = null;
|
||||||
|
|
||||||
constructor(
|
constructor(options: SessionOptions) {
|
||||||
clientId: string,
|
const {
|
||||||
onMessage: (msg: SessionOutboundMessage) => void,
|
clientId,
|
||||||
logger: pino.Logger,
|
onMessage,
|
||||||
downloadTokenStore: DownloadTokenStore,
|
logger,
|
||||||
pushTokenStore: PushTokenStore,
|
downloadTokenStore,
|
||||||
paseoHome: string,
|
pushTokenStore,
|
||||||
agentManager: AgentManager,
|
paseoHome,
|
||||||
agentStorage: AgentStorage,
|
agentManager,
|
||||||
createAgentMcpTransport: AgentMcpTransportFactory,
|
agentStorage,
|
||||||
stt: SpeechToTextProvider | null,
|
createAgentMcpTransport,
|
||||||
tts: TextToSpeechProvider | null,
|
stt,
|
||||||
terminalManager: TerminalManager | null,
|
tts,
|
||||||
voice?: {
|
terminalManager,
|
||||||
voiceLlmProvider?: AgentProvider | null;
|
voice,
|
||||||
voiceLlmModeId?: string | null;
|
voiceBridge,
|
||||||
voiceLlmProviderExplicit?: boolean;
|
dictation,
|
||||||
voiceLlmModel?: string | null;
|
} = options;
|
||||||
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
|
|
||||||
},
|
|
||||||
voiceBridge?: {
|
|
||||||
registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void;
|
|
||||||
unregisterVoiceSpeakHandler?: (agentId: string) => void;
|
|
||||||
registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void;
|
|
||||||
unregisterVoiceCallerContext?: (agentId: string) => void;
|
|
||||||
},
|
|
||||||
dictation?: {
|
|
||||||
finalTimeoutMs?: number;
|
|
||||||
stt?: SpeechToTextProvider | null;
|
|
||||||
localModels?: {
|
|
||||||
modelsDir: string;
|
|
||||||
defaultModelIds: SherpaOnnxModelId[];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
) {
|
|
||||||
this.clientId = clientId;
|
this.clientId = clientId;
|
||||||
this.sessionId = uuidv4();
|
this.sessionId = uuidv4();
|
||||||
this.onMessage = onMessage;
|
this.onMessage = onMessage;
|
||||||
@@ -1148,12 +1162,16 @@ export class Session {
|
|||||||
voiceConversationId: string,
|
voiceConversationId: string,
|
||||||
requestId: string
|
requestId: string
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
this.voiceAssistantAgentId = voiceConversationId;
|
const normalizedVoiceConversationId = this.parseVoiceConversationId(
|
||||||
|
voiceConversationId,
|
||||||
|
"load_voice_conversation_request"
|
||||||
|
);
|
||||||
|
this.voiceAssistantAgentId = normalizedVoiceConversationId;
|
||||||
|
|
||||||
this.emit({
|
this.emit({
|
||||||
type: "voice_conversation_loaded",
|
type: "voice_conversation_loaded",
|
||||||
payload: {
|
payload: {
|
||||||
voiceConversationId,
|
voiceConversationId: normalizedVoiceConversationId,
|
||||||
messageCount: 0,
|
messageCount: 0,
|
||||||
requestId,
|
requestId,
|
||||||
},
|
},
|
||||||
@@ -1216,13 +1234,17 @@ export class Session {
|
|||||||
voiceConversationId: string,
|
voiceConversationId: string,
|
||||||
requestId: string
|
requestId: string
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
const normalizedVoiceConversationId = this.parseVoiceConversationId(
|
||||||
|
voiceConversationId,
|
||||||
|
"delete_voice_conversation_request"
|
||||||
|
);
|
||||||
try {
|
try {
|
||||||
const record = await this.agentStorage.get(voiceConversationId);
|
const record = await this.agentStorage.get(normalizedVoiceConversationId);
|
||||||
if (!record || record.labels?.surface !== "voice") {
|
if (!record || record.labels?.surface !== "voice") {
|
||||||
this.emit({
|
this.emit({
|
||||||
type: "delete_voice_conversation_response",
|
type: "delete_voice_conversation_response",
|
||||||
payload: {
|
payload: {
|
||||||
voiceConversationId,
|
voiceConversationId: normalizedVoiceConversationId,
|
||||||
success: false,
|
success: false,
|
||||||
error: "Voice conversation not found",
|
error: "Voice conversation not found",
|
||||||
requestId,
|
requestId,
|
||||||
@@ -1231,32 +1253,32 @@ export class Session {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const live = this.agentManager.getAgent(voiceConversationId);
|
const live = this.agentManager.getAgent(normalizedVoiceConversationId);
|
||||||
if (live) {
|
if (live) {
|
||||||
await this.agentManager.closeAgent(voiceConversationId);
|
await this.agentManager.closeAgent(normalizedVoiceConversationId);
|
||||||
}
|
}
|
||||||
await this.agentStorage.remove(voiceConversationId);
|
await this.agentStorage.remove(normalizedVoiceConversationId);
|
||||||
this.emit({
|
this.emit({
|
||||||
type: "delete_voice_conversation_response",
|
type: "delete_voice_conversation_response",
|
||||||
payload: {
|
payload: {
|
||||||
voiceConversationId,
|
voiceConversationId: normalizedVoiceConversationId,
|
||||||
success: true,
|
success: true,
|
||||||
requestId,
|
requestId,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{ voiceConversationId },
|
{ voiceConversationId: normalizedVoiceConversationId },
|
||||||
`Deleted voice conversation ${voiceConversationId}`
|
`Deleted voice conversation ${normalizedVoiceConversationId}`
|
||||||
);
|
);
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
this.sessionLogger.error(
|
this.sessionLogger.error(
|
||||||
{ err: error, voiceConversationId },
|
{ err: error, voiceConversationId: normalizedVoiceConversationId },
|
||||||
`Failed to delete voice conversation ${voiceConversationId}`
|
`Failed to delete voice conversation ${normalizedVoiceConversationId}`
|
||||||
);
|
);
|
||||||
this.emit({
|
this.emit({
|
||||||
type: "delete_voice_conversation_response",
|
type: "delete_voice_conversation_response",
|
||||||
payload: {
|
payload: {
|
||||||
voiceConversationId,
|
voiceConversationId: normalizedVoiceConversationId,
|
||||||
success: false,
|
success: false,
|
||||||
error: error.message,
|
error: error.message,
|
||||||
requestId,
|
requestId,
|
||||||
@@ -1393,15 +1415,22 @@ export class Session {
|
|||||||
voiceConversationId?: string
|
voiceConversationId?: string
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (enabled) {
|
if (enabled) {
|
||||||
if (!voiceConversationId || voiceConversationId.trim().length === 0) {
|
const candidateVoiceConversationId =
|
||||||
this.sessionLogger.warn("set_voice_conversation missing voiceConversationId; ignoring");
|
voiceConversationId && voiceConversationId.trim().length > 0
|
||||||
return;
|
? voiceConversationId
|
||||||
}
|
: uuidv4();
|
||||||
|
const normalizedVoiceConversationId = this.parseVoiceConversationId(
|
||||||
|
candidateVoiceConversationId,
|
||||||
|
"set_voice_conversation"
|
||||||
|
);
|
||||||
|
|
||||||
this.isVoiceMode = true;
|
this.isVoiceMode = true;
|
||||||
this.voiceAssistantAgentId = voiceConversationId;
|
this.voiceAssistantAgentId = normalizedVoiceConversationId;
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
|
{
|
||||||
|
voiceAssistantAgentId: this.voiceAssistantAgentId,
|
||||||
|
generatedConversationId: !voiceConversationId,
|
||||||
|
},
|
||||||
"Voice conversation enabled (agent-backed)"
|
"Voice conversation enabled (agent-backed)"
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
@@ -1414,6 +1443,14 @@ export class Session {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private parseVoiceConversationId(rawId: string, source: string): string {
|
||||||
|
const parsed = VoiceConversationIdSchema.safeParse(rawId.trim());
|
||||||
|
if (!parsed.success) {
|
||||||
|
throw new Error(`${source}: voiceConversationId must be a UUID`);
|
||||||
|
}
|
||||||
|
return parsed.data;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handle text message to agent (with optional image attachments)
|
* Handle text message to agent (with optional image attachments)
|
||||||
*/
|
*/
|
||||||
@@ -4259,7 +4296,7 @@ export class Session {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Process through LLM (voice path is agent-backed only)
|
// Process through LLM (voice path is agent-backed only)
|
||||||
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, text);
|
this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, text);
|
||||||
await this.currentStreamPromise;
|
await this.currentStreamPromise;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4528,7 +4565,7 @@ export class Session {
|
|||||||
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
|
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
|
||||||
this.clearSpeechInProgress("transcription complete");
|
this.clearSpeechInProgress("transcription complete");
|
||||||
this.setPhase("llm");
|
this.setPhase("llm");
|
||||||
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, result.text);
|
this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, result.text);
|
||||||
await this.currentStreamPromise;
|
await this.currentStreamPromise;
|
||||||
this.setPhase("idle");
|
this.setPhase("idle");
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
@@ -4601,30 +4638,47 @@ export class Session {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
this.voiceAssistantAgentId = created.id;
|
this.voiceAssistantAgentId = created.id;
|
||||||
|
let speakHandlerRegistered = false;
|
||||||
|
let callerContextRegistered = false;
|
||||||
|
|
||||||
this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => {
|
try {
|
||||||
const abortSignal = signal ?? this.abortController.signal;
|
this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => {
|
||||||
await this.ttsManager.generateAndWaitForPlayback(
|
const abortSignal = signal ?? this.abortController.signal;
|
||||||
text,
|
await this.ttsManager.generateAndWaitForPlayback(
|
||||||
(msg) => this.emit(msg),
|
text,
|
||||||
abortSignal,
|
(msg) => this.emit(msg),
|
||||||
true
|
abortSignal,
|
||||||
);
|
true
|
||||||
this.emit({
|
);
|
||||||
type: "activity_log",
|
this.emit({
|
||||||
payload: {
|
type: "activity_log",
|
||||||
id: uuidv4(),
|
payload: {
|
||||||
timestamp: new Date(),
|
id: uuidv4(),
|
||||||
type: "assistant",
|
timestamp: new Date(),
|
||||||
content: text,
|
type: "assistant",
|
||||||
},
|
content: text,
|
||||||
|
},
|
||||||
|
});
|
||||||
});
|
});
|
||||||
});
|
speakHandlerRegistered = true;
|
||||||
this.registerVoiceCallerContext?.(created.id, {
|
|
||||||
childAgentDefaultLabels: { ui: "true" },
|
this.registerVoiceCallerContext?.(created.id, {
|
||||||
allowCustomCwd: true,
|
childAgentDefaultLabels: { ui: "true" },
|
||||||
enableVoiceTools: true,
|
allowCustomCwd: true,
|
||||||
});
|
enableVoiceTools: true,
|
||||||
|
});
|
||||||
|
callerContextRegistered = true;
|
||||||
|
} catch (error) {
|
||||||
|
if (speakHandlerRegistered) {
|
||||||
|
this.unregisterVoiceSpeakHandler?.(created.id);
|
||||||
|
}
|
||||||
|
if (callerContextRegistered) {
|
||||||
|
this.unregisterVoiceCallerContext?.(created.id);
|
||||||
|
}
|
||||||
|
await this.agentManager.closeAgent(created.id).catch(() => undefined);
|
||||||
|
this.voiceAssistantAgentId = null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{
|
{
|
||||||
@@ -4647,20 +4701,7 @@ export class Session {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private shouldAllowVoicePermission(request: AgentPermissionRequest): boolean {
|
private shouldAllowVoicePermission(request: AgentPermissionRequest): boolean {
|
||||||
const name = request.name.toLowerCase();
|
return isVoicePermissionAllowed(request);
|
||||||
if (name.includes("mcp") || name.includes("paseo") || name.includes("speak")) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (name === "codextool") {
|
|
||||||
const metadata = request.metadata ?? {};
|
|
||||||
const rawQuestions = metadata.questions;
|
|
||||||
if (Array.isArray(rawQuestions)) {
|
|
||||||
const text = JSON.stringify(rawQuestions).toLowerCase();
|
|
||||||
return text.includes("mcp") || text.includes("paseo") || text.includes("speak");
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async processWithVoiceAgent(userText: string): Promise<void> {
|
private async processWithVoiceAgent(userText: string): Promise<void> {
|
||||||
@@ -4733,10 +4774,10 @@ export class Session {
|
|||||||
/**
|
/**
|
||||||
* Process user message through LLM with streaming and tool execution
|
* Process user message through LLM with streaming and tool execution
|
||||||
*/
|
*/
|
||||||
private async processWithLLM(enableTTS: boolean, latestUserText?: string): Promise<void> {
|
private async processVoiceTurn(enableTTS: boolean, latestUserText?: string): Promise<void> {
|
||||||
try {
|
try {
|
||||||
if (!enableTTS) {
|
if (!enableTTS) {
|
||||||
this.sessionLogger.warn("Ignoring non-voice processWithLLM call; voice is agent-only");
|
this.sessionLogger.warn("Ignoring non-voice processVoiceTurn call; voice is agent-only");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const normalized = (latestUserText ?? "").trim();
|
const normalized = (latestUserText ?? "").trim();
|
||||||
|
|||||||
+2
-1
@@ -2,6 +2,7 @@ import { describe, expect, test } from "vitest";
|
|||||||
import { mkdtempSync } from "node:fs";
|
import { mkdtempSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
import pino from "pino";
|
import pino from "pino";
|
||||||
|
|
||||||
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
|
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
|
||||||
@@ -186,7 +187,7 @@ describe("speech models (download E2E)", () => {
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`);
|
await ctx.client.setVoiceConversation(true, randomUUID());
|
||||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||||
const isLast = offset + chunkBytes >= pcm16.length;
|
const isLast = offset + chunkBytes >= pcm16.length;
|
||||||
|
|||||||
@@ -5,14 +5,14 @@ import type { TTSConfig } from "./providers/openai/tts.js";
|
|||||||
import type { PersistedConfig } from "../persisted-config.js";
|
import type { PersistedConfig } from "../persisted-config.js";
|
||||||
import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
|
import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
|
||||||
import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/sherpa/model-catalog.js";
|
import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/sherpa/model-catalog.js";
|
||||||
|
import {
|
||||||
const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8";
|
DEFAULT_LOCAL_STT_MODEL,
|
||||||
const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8";
|
DEFAULT_LOCAL_TTS_MODEL,
|
||||||
|
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
|
||||||
|
type SpeechProviderId,
|
||||||
|
} from "./speech-types.js";
|
||||||
const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
|
const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
|
||||||
const DEFAULT_OPENAI_TTS_MODEL: TTSConfig["model"] = "tts-1";
|
const DEFAULT_OPENAI_TTS_MODEL: TTSConfig["model"] = "tts-1";
|
||||||
const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
|
||||||
|
|
||||||
type SpeechProviderId = "openai" | "local";
|
|
||||||
|
|
||||||
function parseSpeechProviderId(value: unknown): SpeechProviderId | null {
|
function parseSpeechProviderId(value: unknown): SpeechProviderId | null {
|
||||||
if (typeof value !== "string") return null;
|
if (typeof value !== "string") return null;
|
||||||
|
|||||||
@@ -25,12 +25,12 @@ import {
|
|||||||
type SherpaOnnxModelId,
|
type SherpaOnnxModelId,
|
||||||
} from "./providers/local/sherpa/model-catalog.js";
|
} from "./providers/local/sherpa/model-catalog.js";
|
||||||
import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js";
|
import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js";
|
||||||
|
import {
|
||||||
type SpeechProviderId = "openai" | "local";
|
DEFAULT_LOCAL_STT_MODEL,
|
||||||
|
DEFAULT_LOCAL_TTS_MODEL,
|
||||||
const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8";
|
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
|
||||||
const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8";
|
type SpeechProviderId,
|
||||||
const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
} from "./speech-types.js";
|
||||||
|
|
||||||
type LocalSttEngine =
|
type LocalSttEngine =
|
||||||
| { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
|
| { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import type {
|
||||||
|
LocalSttModelId,
|
||||||
|
LocalTtsModelId,
|
||||||
|
} from "./providers/local/sherpa/model-catalog.js";
|
||||||
|
|
||||||
|
export type SpeechProviderId = "openai" | "local";
|
||||||
|
|
||||||
|
export const DEFAULT_LOCAL_STT_MODEL: LocalSttModelId = "parakeet-tdt-0.6b-v3-int8";
|
||||||
|
export const DEFAULT_LOCAL_TTS_MODEL: LocalTtsModelId = "pocket-tts-onnx-int8";
|
||||||
|
export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
|
||||||
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
|
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
|
||||||
|
|
||||||
@@ -64,7 +65,7 @@ function waitForSignal<T>(
|
|||||||
test(
|
test(
|
||||||
"routes voice turns through local agent speak tool",
|
"routes voice turns through local agent speak tool",
|
||||||
async () => {
|
async () => {
|
||||||
await ctx.client.setVoiceConversation(true, `voice-local-agent-${Date.now()}`);
|
await ctx.client.setVoiceConversation(true, randomUUID());
|
||||||
|
|
||||||
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
|
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
|
||||||
const offAudio = ctx.client.on("audio_output", (msg) => {
|
const offAudio = ctx.client.on("audio_output", (msg) => {
|
||||||
|
|||||||
@@ -221,17 +221,35 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro
|
|||||||
|
|
||||||
const socket = net.createConnection(parsed.socketPath);
|
const socket = net.createConnection(parsed.socketPath);
|
||||||
const stdioTransport = new StdioServerTransport(process.stdin, process.stdout);
|
const stdioTransport = new StdioServerTransport(process.stdin, process.stdout);
|
||||||
let ready = false;
|
|
||||||
let socketBuffer = "";
|
let socketBuffer = "";
|
||||||
|
let stdioStarted = false;
|
||||||
|
const pendingMcpMessages: JSONRPCMessage[] = [];
|
||||||
|
|
||||||
|
let resolveReady: (() => void) | null = null;
|
||||||
|
let rejectReady: ((error: Error) => void) | null = null;
|
||||||
|
const readyPromise = new Promise<void>((resolve, reject) => {
|
||||||
|
resolveReady = () => {
|
||||||
|
resolveReady = null;
|
||||||
|
rejectReady = null;
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
rejectReady = (error: Error) => {
|
||||||
|
resolveReady = null;
|
||||||
|
rejectReady = null;
|
||||||
|
reject(error);
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const failReady = (message: string) => {
|
||||||
|
if (rejectReady) {
|
||||||
|
rejectReady(new Error(message));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const sendEnvelope = (payload: BridgeEnvelope) => {
|
const sendEnvelope = (payload: BridgeEnvelope) => {
|
||||||
socket.write(encodeEnvelope(payload));
|
socket.write(encodeEnvelope(payload));
|
||||||
};
|
};
|
||||||
|
|
||||||
const closeWithError = (message: string): never => {
|
|
||||||
throw new Error(message);
|
|
||||||
};
|
|
||||||
|
|
||||||
socket.on("data", (chunk) => {
|
socket.on("data", (chunk) => {
|
||||||
socketBuffer += chunk.toString("utf8");
|
socketBuffer += chunk.toString("utf8");
|
||||||
while (true) {
|
while (true) {
|
||||||
@@ -241,23 +259,41 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro
|
|||||||
socketBuffer = socketBuffer.slice(newlineIndex + 1);
|
socketBuffer = socketBuffer.slice(newlineIndex + 1);
|
||||||
if (!line) continue;
|
if (!line) continue;
|
||||||
|
|
||||||
const envelope = parseEnvelope(line);
|
let envelope: BridgeEnvelope;
|
||||||
|
try {
|
||||||
|
envelope = parseEnvelope(line);
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
bridgeLogger.error({ err: error }, "Failed to parse voice MCP bridge envelope");
|
||||||
|
failReady(`Failed to parse voice MCP bridge envelope: ${message}`);
|
||||||
|
socket.destroy();
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (envelope.type === "ready") {
|
if (envelope.type === "ready") {
|
||||||
ready = true;
|
resolveReady?.();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (envelope.type === "error") {
|
if (envelope.type === "error") {
|
||||||
|
failReady(`Voice MCP bridge error: ${envelope.message}`);
|
||||||
socket.destroy();
|
socket.destroy();
|
||||||
closeWithError(`Voice MCP bridge error: ${envelope.message}`);
|
return;
|
||||||
}
|
}
|
||||||
if (envelope.type === "mcp") {
|
if (envelope.type === "mcp") {
|
||||||
void stdioTransport.send(envelope.message);
|
if (!stdioStarted) {
|
||||||
|
pendingMcpMessages.push(envelope.message);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
void stdioTransport.send(envelope.message).catch((error) => {
|
||||||
|
bridgeLogger.error({ err: error }, "Failed to forward MCP message to stdio transport");
|
||||||
|
socket.destroy();
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
socket.on("error", (error) => {
|
socket.on("error", (error) => {
|
||||||
bridgeLogger.error({ err: error }, "Voice MCP bridge socket client error");
|
bridgeLogger.error({ err: error }, "Voice MCP bridge socket client error");
|
||||||
|
failReady(`Voice MCP bridge socket client error: ${error.message}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
@@ -267,21 +303,13 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro
|
|||||||
|
|
||||||
sendEnvelope({ type: "init", callerAgentId: parsed.callerAgentId });
|
sendEnvelope({ type: "init", callerAgentId: parsed.callerAgentId });
|
||||||
|
|
||||||
// Wait for bridge readiness before forwarding stdio messages.
|
const readyTimeoutPromise = new Promise<void>((_, reject) => {
|
||||||
await new Promise<void>((resolve, reject) => {
|
const timeout = setTimeout(() => {
|
||||||
const deadline = setTimeout(() => {
|
|
||||||
reject(new Error("Timed out waiting for voice MCP bridge initialization"));
|
reject(new Error("Timed out waiting for voice MCP bridge initialization"));
|
||||||
}, 10000);
|
}, 10000);
|
||||||
const checkReady = () => {
|
readyPromise.finally(() => clearTimeout(timeout)).catch(() => undefined);
|
||||||
if (ready) {
|
|
||||||
clearTimeout(deadline);
|
|
||||||
resolve();
|
|
||||||
} else {
|
|
||||||
setTimeout(checkReady, 25);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
checkReady();
|
|
||||||
});
|
});
|
||||||
|
await Promise.race([readyPromise, readyTimeoutPromise]);
|
||||||
|
|
||||||
stdioTransport.onmessage = (message) => {
|
stdioTransport.onmessage = (message) => {
|
||||||
sendEnvelope({ type: "mcp", message });
|
sendEnvelope({ type: "mcp", message });
|
||||||
@@ -295,6 +323,10 @@ export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Pro
|
|||||||
};
|
};
|
||||||
|
|
||||||
await stdioTransport.start();
|
await stdioTransport.start();
|
||||||
|
stdioStarted = true;
|
||||||
|
for (const message of pendingMcpMessages) {
|
||||||
|
await stdioTransport.send(message);
|
||||||
|
}
|
||||||
|
|
||||||
await new Promise<void>((resolve) => {
|
await new Promise<void>((resolve) => {
|
||||||
socket.once("close", () => resolve());
|
socket.once("close", () => resolve());
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
import { describe, expect, test } from "vitest";
|
||||||
|
|
||||||
|
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
|
||||||
|
import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
|
||||||
|
|
||||||
|
function buildRequest(partial: Partial<AgentPermissionRequest>): AgentPermissionRequest {
|
||||||
|
return {
|
||||||
|
id: "req-1",
|
||||||
|
provider: "codex",
|
||||||
|
name: "unknown",
|
||||||
|
kind: "tool",
|
||||||
|
...partial,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("isVoicePermissionAllowed", () => {
|
||||||
|
test("allows speak tool", () => {
|
||||||
|
const result = isVoicePermissionAllowed(
|
||||||
|
buildRequest({ name: "speak" })
|
||||||
|
);
|
||||||
|
expect(result).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("allows explicit MCP/paseo tool names", () => {
|
||||||
|
expect(
|
||||||
|
isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))
|
||||||
|
).toBe(true);
|
||||||
|
expect(
|
||||||
|
isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("denies non-tool permission kinds", () => {
|
||||||
|
const result = isVoicePermissionAllowed(
|
||||||
|
buildRequest({ kind: "mode", name: "mcp__paseo__create_agent" })
|
||||||
|
);
|
||||||
|
expect(result).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("allows codextool only when metadata references MCP/paseo", () => {
|
||||||
|
const allowed = isVoicePermissionAllowed(
|
||||||
|
buildRequest({
|
||||||
|
name: "codextool",
|
||||||
|
metadata: {
|
||||||
|
questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
);
|
||||||
|
expect(allowed).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("denies codextool when metadata includes shell-like operations", () => {
|
||||||
|
const denied = isVoicePermissionAllowed(
|
||||||
|
buildRequest({
|
||||||
|
name: "codextool",
|
||||||
|
metadata: {
|
||||||
|
questions: [{ question: "Allow codextool to execute shell command?" }],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
);
|
||||||
|
expect(denied).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
|
||||||
|
|
||||||
|
const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"];
|
||||||
|
const DENIED_TOKEN_SET = [
|
||||||
|
"bash",
|
||||||
|
"shell",
|
||||||
|
"terminal",
|
||||||
|
"command",
|
||||||
|
"execute",
|
||||||
|
"edit",
|
||||||
|
"write",
|
||||||
|
"read",
|
||||||
|
"fetch",
|
||||||
|
"http",
|
||||||
|
"web",
|
||||||
|
];
|
||||||
|
|
||||||
|
function containsAny(text: string, tokens: readonly string[]): boolean {
|
||||||
|
return tokens.some((token) => text.includes(token));
|
||||||
|
}
|
||||||
|
|
||||||
|
function stringifyMetadata(metadata: unknown): string {
|
||||||
|
try {
|
||||||
|
return JSON.stringify(metadata ?? {})?.toLowerCase() ?? "";
|
||||||
|
} catch {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Voice assistant policy: allow only MCP/paseo/speak tool requests.
|
||||||
|
* All non-tool permission requests are denied.
|
||||||
|
*/
|
||||||
|
export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean {
|
||||||
|
if (request.kind !== "tool") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalizedName = request.name.trim().toLowerCase();
|
||||||
|
if (!normalizedName) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalizedName === "speak") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalizedName !== "codextool") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const metadataText = stringifyMetadata({
|
||||||
|
metadata: request.metadata ?? null,
|
||||||
|
input: request.input ?? null,
|
||||||
|
});
|
||||||
|
if (!metadataText) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const mentionsAllowedTooling = containsAny(metadataText, ALLOWED_TOKEN_SET);
|
||||||
|
const mentionsDeniedTooling = containsAny(metadataText, DENIED_TOKEN_SET);
|
||||||
|
return mentionsAllowedTooling && !mentionsDeniedTooling;
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
export type VoiceSpeakHandler = (params: {
|
||||||
|
text: string;
|
||||||
|
callerAgentId: string;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}) => Promise<void>;
|
||||||
|
|
||||||
|
export type VoiceCallerContext = {
|
||||||
|
childAgentDefaultLabels?: Record<string, string>;
|
||||||
|
lockedCwd?: string;
|
||||||
|
allowCustomCwd?: boolean;
|
||||||
|
enableVoiceTools?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type VoiceMcpStdioConfig = {
|
||||||
|
command: string;
|
||||||
|
baseArgs: string[];
|
||||||
|
env?: Record<string, string>;
|
||||||
|
};
|
||||||
@@ -21,13 +21,13 @@ import { PushTokenStore } from "./push/token-store.js";
|
|||||||
import { PushService } from "./push/push-service.js";
|
import { PushService } from "./push/push-service.js";
|
||||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
|
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
|
||||||
import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js";
|
import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js";
|
||||||
|
import type {
|
||||||
|
VoiceCallerContext,
|
||||||
|
VoiceMcpStdioConfig,
|
||||||
|
VoiceSpeakHandler,
|
||||||
|
} from "./voice-types.js";
|
||||||
|
|
||||||
export type AgentMcpTransportFactory = () => Promise<Transport>;
|
export type AgentMcpTransportFactory = () => Promise<Transport>;
|
||||||
type VoiceMcpStdioConfig = {
|
|
||||||
command: string;
|
|
||||||
baseArgs: string[];
|
|
||||||
env?: Record<string, string>;
|
|
||||||
};
|
|
||||||
|
|
||||||
type WebSocketServerConfig = {
|
type WebSocketServerConfig = {
|
||||||
allowedOrigins: Set<string>;
|
allowedOrigins: Set<string>;
|
||||||
@@ -91,17 +91,9 @@ export class VoiceAssistantWebSocketServer {
|
|||||||
} | null;
|
} | null;
|
||||||
private readonly voiceSpeakHandlers = new Map<
|
private readonly voiceSpeakHandlers = new Map<
|
||||||
string,
|
string,
|
||||||
(params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>
|
VoiceSpeakHandler
|
||||||
>();
|
|
||||||
private readonly voiceCallerContexts = new Map<
|
|
||||||
string,
|
|
||||||
{
|
|
||||||
childAgentDefaultLabels?: Record<string, string>;
|
|
||||||
lockedCwd?: string;
|
|
||||||
allowCustomCwd?: boolean;
|
|
||||||
enableVoiceTools?: boolean;
|
|
||||||
}
|
|
||||||
>();
|
>();
|
||||||
|
private readonly voiceCallerContexts = new Map<string, VoiceCallerContext>();
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
server: HTTPServer,
|
server: HTTPServer,
|
||||||
@@ -236,23 +228,23 @@ export class VoiceAssistantWebSocketServer {
|
|||||||
const clientId = `client-${++this.clientIdCounter}`;
|
const clientId = `client-${++this.clientIdCounter}`;
|
||||||
const connectionLogger = this.logger.child({ clientId });
|
const connectionLogger = this.logger.child({ clientId });
|
||||||
|
|
||||||
const session = new Session(
|
const session = new Session({
|
||||||
clientId,
|
clientId,
|
||||||
(msg) => {
|
onMessage: (msg) => {
|
||||||
this.sendToClient(ws, wrapSessionMessage(msg));
|
this.sendToClient(ws, wrapSessionMessage(msg));
|
||||||
},
|
},
|
||||||
connectionLogger.child({ module: "session" }),
|
logger: connectionLogger.child({ module: "session" }),
|
||||||
this.downloadTokenStore,
|
downloadTokenStore: this.downloadTokenStore,
|
||||||
this.pushTokenStore,
|
pushTokenStore: this.pushTokenStore,
|
||||||
this.paseoHome,
|
paseoHome: this.paseoHome,
|
||||||
this.agentManager,
|
agentManager: this.agentManager,
|
||||||
this.agentStorage,
|
agentStorage: this.agentStorage,
|
||||||
this.createAgentMcpTransport,
|
createAgentMcpTransport: this.createAgentMcpTransport,
|
||||||
this.stt,
|
stt: this.stt,
|
||||||
this.tts,
|
tts: this.tts,
|
||||||
this.terminalManager,
|
terminalManager: this.terminalManager,
|
||||||
this.voice ?? undefined,
|
voice: this.voice ?? undefined,
|
||||||
{
|
voiceBridge: {
|
||||||
registerVoiceSpeakHandler: (agentId, handler) => {
|
registerVoiceSpeakHandler: (agentId, handler) => {
|
||||||
this.voiceSpeakHandlers.set(agentId, handler);
|
this.voiceSpeakHandlers.set(agentId, handler);
|
||||||
},
|
},
|
||||||
@@ -266,8 +258,8 @@ export class VoiceAssistantWebSocketServer {
|
|||||||
this.voiceCallerContexts.delete(agentId);
|
this.voiceCallerContexts.delete(agentId);
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
this.dictation ?? undefined
|
dictation: this.dictation ?? undefined,
|
||||||
);
|
});
|
||||||
|
|
||||||
this.sessions.set(ws, session);
|
this.sessions.set(ws, session);
|
||||||
|
|
||||||
@@ -306,18 +298,13 @@ export class VoiceAssistantWebSocketServer {
|
|||||||
|
|
||||||
public resolveVoiceSpeakHandler(
|
public resolveVoiceSpeakHandler(
|
||||||
callerAgentId: string
|
callerAgentId: string
|
||||||
): ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>) | null {
|
): VoiceSpeakHandler | null {
|
||||||
return this.voiceSpeakHandlers.get(callerAgentId) ?? null;
|
return this.voiceSpeakHandlers.get(callerAgentId) ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public resolveVoiceCallerContext(
|
public resolveVoiceCallerContext(
|
||||||
callerAgentId: string
|
callerAgentId: string
|
||||||
): {
|
): VoiceCallerContext | null {
|
||||||
childAgentDefaultLabels?: Record<string, string>;
|
|
||||||
lockedCwd?: string;
|
|
||||||
allowCustomCwd?: boolean;
|
|
||||||
enableVoiceTools?: boolean;
|
|
||||||
} | null {
|
|
||||||
return this.voiceCallerContexts.get(callerAgentId) ?? null;
|
return this.voiceCallerContexts.get(callerAgentId) ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user