Update files

2026-02-07 12:52:22 +07:00
parent 4a9cf9e90f
commit 6cfae77a71
24 changed files with 1312 additions and 443 deletions
@@ -0,0 +1,62 @@
+import net from "node:net";
+
+function parseArgs(argv) {
+  let socketPath = null;
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--socket") {
+      socketPath = argv[index + 1] ?? null;
+      index += 1;
+    }
+  }
+
+  if (!socketPath || !socketPath.trim()) {
+    throw new Error("Missing required --socket <path>");
+  }
+
+  return { socketPath: socketPath.trim() };
+}
+
+async function main() {
+  const { socketPath } = parseArgs(process.argv.slice(2));
+  const socket = net.createConnection(socketPath);
+
+  socket.on("error", (error) => {
+    process.stderr.write(`MCP stdio-socket bridge error: ${error.message}\n`);
+    process.exitCode = 1;
+  });
+
+  process.stdin.on("error", (error) => {
+    process.stderr.write(`MCP stdio-socket bridge stdin error: ${error.message}\n`);
+    socket.destroy();
+    process.exitCode = 1;
+  });
+
+  process.stdout.on("error", (error) => {
+    process.stderr.write(`MCP stdio-socket bridge stdout error: ${error.message}\n`);
+    socket.destroy();
+    process.exitCode = 1;
+  });
+
+  await new Promise((resolve, reject) => {
+    socket.once("connect", resolve);
+    socket.once("error", reject);
+  });
+
+  process.stdin.pipe(socket);
+  socket.pipe(process.stdout);
+
+  await new Promise((resolve) => {
+    socket.once("close", resolve);
+    process.stdin.once("end", () => {
+      socket.end();
+      resolve();
+    });
+  });
+}
+
+main().catch((error) => {
+  process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+  process.exit(1);
+});
@@ -0,0 +1,172 @@
+import { randomUUID } from "node:crypto";
+import pino from "pino";
+
+import { DaemonClient } from "../src/server/test-utils/daemon-client.js";
+import { OpenAITTS } from "../src/server/speech/providers/openai/tts.js";
+
+type RoundTripResult = {
+  voiceAgentId: string;
+  speakToolCalls: number;
+  audioChunks: number;
+};
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function streamToBuffer(stream: AsyncIterable<unknown>): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as ArrayBufferLike));
+  }
+  return Buffer.concat(chunks);
+}
+
+async function synthesizeInput(text: string, logger: pino.Logger): Promise<Buffer> {
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) {
+    throw new Error("OPENAI_API_KEY is required for voice-main-daemon-e2e");
+  }
+
+  const tts = new OpenAITTS(
+    {
+      apiKey,
+      responseFormat: "pcm",
+      voice: "alloy",
+    },
+    logger
+  );
+
+  const generated = await tts.synthesizeSpeech(text);
+  return streamToBuffer(generated.stream as AsyncIterable<unknown>);
+}
+
+async function runVoiceRoundTrip(params: {
+  daemonUrl: string;
+  voiceAgentId: string;
+  speechText: string;
+  timeoutMs: number;
+}): Promise<RoundTripResult> {
+  const client = new DaemonClient({ url: `${params.daemonUrl}/ws` });
+  await client.connect();
+  client.subscribeAgentUpdates({ subscriptionId: `voice-e2e-${randomUUID()}` });
+
+  const mode = await client.setVoiceMode(true, params.voiceAgentId);
+  if (!mode.accepted) {
+    throw new Error(`set_voice_mode rejected: ${mode.error ?? "unknown error"}`);
+  }
+  if (!mode.voiceAgentId) {
+    throw new Error("set_voice_mode returned null voiceAgentId");
+  }
+
+  const activeVoiceAgentId = mode.voiceAgentId;
+  const pcm = await synthesizeInput(params.speechText, pino({ level: "warn" }));
+
+  let audioChunks = 0;
+  let speakToolCalls = 0;
+  let sawLastAudio = false;
+
+  const done = new Promise<void>((resolve) => {
+    const offAudio = client.on("audio_output", (msg) => {
+      if (msg.type !== "audio_output") return;
+      audioChunks += 1;
+      if (msg.payload.isLastChunk) {
+        sawLastAudio = true;
+        offAudio();
+        resolve();
+      }
+    });
+  });
+
+  const offStream = client.on("agent_stream", (msg) => {
+    if (msg.type !== "agent_stream") return;
+    if (msg.payload.agentId !== activeVoiceAgentId) return;
+    if (msg.payload.event.type !== "timeline") return;
+    const item = msg.payload.event.item;
+    if (item.type !== "tool_call") return;
+    if (typeof item.name === "string" && item.name.toLowerCase().includes("speak")) {
+      speakToolCalls += 1;
+    }
+  });
+
+  const chunkBytes = 4800;
+  for (let offset = 0; offset < pcm.length; offset += chunkBytes) {
+    const chunk = pcm.subarray(offset, Math.min(pcm.length, offset + chunkBytes));
+    const isLast = offset + chunkBytes >= pcm.length;
+    await client.sendVoiceAudioChunk(
+      chunk.toString("base64"),
+      "audio/pcm;rate=24000;bits=16",
+      isLast
+    );
+  }
+
+  try {
+    await Promise.race([
+      done,
+      (async () => {
+        await sleep(params.timeoutMs);
+        throw new Error("Timed out waiting for audio_output");
+      })(),
+    ]);
+  } finally {
+    offStream();
+    await client.setVoiceMode(false).catch(() => undefined);
+    await client.close().catch(() => undefined);
+  }
+
+  if (!sawLastAudio) {
+    throw new Error("No final audio chunk observed");
+  }
+  if (audioChunks === 0) {
+    throw new Error("No audio chunks observed");
+  }
+  if (speakToolCalls === 0) {
+    throw new Error("No speak tool call observed in agent stream");
+  }
+
+  return {
+    voiceAgentId: activeVoiceAgentId,
+    speakToolCalls,
+    audioChunks,
+  };
+}
+
+async function main(): Promise<void> {
+  const daemonUrl = process.env.PASEO_DAEMON_URL ?? "ws://localhost:6767".replace(/^ws/, "http");
+  const timeoutMs = Number(process.env.PASEO_VOICE_E2E_TIMEOUT_MS ?? "120000");
+  const voiceAgentId = randomUUID();
+
+  const newAgentResult = await runVoiceRoundTrip({
+    daemonUrl,
+    voiceAgentId,
+    speechText:
+      "Use the speak tool and say exactly: new voice agent round trip successful.",
+    timeoutMs,
+  });
+
+  const resumedResult = await runVoiceRoundTrip({
+    daemonUrl,
+    voiceAgentId: newAgentResult.voiceAgentId,
+    speechText:
+      "Use the speak tool and say exactly: resumed voice agent round trip successful.",
+    timeoutMs,
+  });
+
+  console.log(
+    JSON.stringify(
+      {
+        ok: true,
+        daemonUrl,
+        newAgentResult,
+        resumedResult,
+      },
+      null,
+      2
+    )
+  );
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});
@@ -0,0 +1,207 @@
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import pino from "pino";
+
+import { AgentManager } from "../src/server/agent/agent-manager.js";
+import { AgentStorage } from "../src/server/agent/agent-storage.js";
+import { buildProviderRegistry, shutdownProviders } from "../src/server/agent/provider-registry.js";
+import type {
+  AgentProvider,
+  AgentSessionConfig,
+  AgentStreamEvent,
+} from "../src/server/agent/agent-sdk-types.js";
+import { buildVoiceAgentMcpServerConfig } from "../src/server/session.js";
+import { createAgentMcpServer } from "../src/server/agent/mcp-server.js";
+import { createVoiceMcpSocketBridgeManager } from "../src/server/voice-mcp-bridge.js";
+
+type CliOptions = {
+  provider: AgentProvider;
+  model?: string;
+  timeoutMs: number;
+};
+
+function parseArgs(argv: string[]): CliOptions {
+  let provider: AgentProvider = "claude";
+  let model: string | undefined;
+  let timeoutMs = 120_000;
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--provider" && argv[i + 1]) {
+      provider = argv[i + 1] as AgentProvider;
+      i += 1;
+      continue;
+    }
+    if (arg === "--model" && argv[i + 1]) {
+      model = argv[i + 1];
+      i += 1;
+      continue;
+    }
+    if (arg === "--timeout-ms" && argv[i + 1]) {
+      timeoutMs = Number(argv[i + 1]) || timeoutMs;
+      i += 1;
+    }
+  }
+
+  return { provider, model, timeoutMs };
+}
+
+async function streamWithTimeout(
+  iterator: AsyncGenerator<AgentStreamEvent>,
+  timeoutMs: number,
+  onEvent: (event: AgentStreamEvent) => Promise<void> | void
+): Promise<void> {
+  const timeout = new Promise<never>((_, reject) => {
+    setTimeout(() => reject(new Error(`Timed out after ${timeoutMs}ms`)), timeoutMs);
+  });
+
+  const consume = (async () => {
+    for await (const event of iterator) {
+      await onEvent(event);
+    }
+  })();
+
+  await Promise.race([consume, timeout]);
+}
+
+async function main(): Promise<void> {
+  const opts = parseArgs(process.argv.slice(2));
+  const logger = pino({ level: "info" });
+  const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+  const serverDir = path.resolve(scriptDir, "..");
+  const tmpRoot = await mkdtemp(path.join(os.tmpdir(), "paseo-voice-mcp-smoke-"));
+  const paseoHome = path.join(tmpRoot, ".paseo");
+  const storageDir = path.join(paseoHome, "agents");
+  const voiceBridgeRuntimeDir = path.join(tmpRoot, "voice-mcp");
+  const voiceWorkspace = path.join(tmpRoot, "voice-agent-workspace");
+  await mkdir(voiceWorkspace, { recursive: true });
+
+  const providerRegistry = buildProviderRegistry(logger);
+  const providerDefinition = providerRegistry[opts.provider];
+  if (!providerDefinition) {
+    throw new Error(`Unsupported provider: ${opts.provider}`);
+  }
+
+  const agentStorage = new AgentStorage(storageDir, logger);
+  const agentManager = new AgentManager({
+    logger,
+    registry: agentStorage,
+    clients: {
+      [opts.provider]: providerDefinition.createClient(logger),
+    },
+  });
+
+  const speakCalls: string[] = [];
+  const resolveSpeakHandler = (callerAgentId: string) => {
+    return async ({ text }: { text: string; callerAgentId: string }) => {
+      speakCalls.push(text);
+      logger.info({ callerAgentId, text }, "Smoke speak handler invoked");
+    };
+  };
+
+  const voiceBridgeManager = createVoiceMcpSocketBridgeManager({
+    runtimeDir: voiceBridgeRuntimeDir,
+    logger,
+    createAgentMcpServerForCaller: async (callerAgentId) => {
+      return createAgentMcpServer({
+        agentManager,
+        agentStorage,
+        paseoHome,
+        callerAgentId,
+        enableVoiceTools: false,
+        resolveSpeakHandler,
+        resolveCallerContext: () => ({
+          childAgentDefaultLabels: { ui: "true" },
+          allowCustomCwd: true,
+          enableVoiceTools: true,
+        }),
+        logger,
+      });
+    },
+  });
+
+  const agentId = randomUUID();
+  const socketPath = await voiceBridgeManager.ensureBridgeForCaller(agentId);
+  const bridgeScriptPath = path.resolve(serverDir, "scripts/mcp-stdio-socket-bridge-cli.mjs");
+  const config: AgentSessionConfig = {
+    provider: opts.provider,
+    cwd: voiceWorkspace,
+    modeId: "default",
+    ...(opts.model ? { model: opts.model } : {}),
+    mcpServers: {
+      paseo: buildVoiceAgentMcpServerConfig({
+        command: process.execPath,
+        baseArgs: [bridgeScriptPath],
+        socketPath,
+        env: {
+          PASEO_HOME: paseoHome,
+        },
+      }),
+    },
+  };
+
+  const prompt = [
+    "You must call the MCP tool named speak.",
+    "Call it exactly once with this exact text: BRIDGE_SMOKE_OK",
+    "Do not call any other tools.",
+    "After the tool call, end your response.",
+  ].join("\n");
+
+  let sawSpeakTimeline = false;
+  let sawTurnFailed = false;
+  let failureError: string | null = null;
+
+  try {
+    const created = await agentManager.createAgent(config, agentId, {
+      labels: { surface: "voice-smoke", ui: "true" },
+    });
+    logger.info({ provider: opts.provider, agentId: created.id }, "Created smoke agent");
+
+    const iterator = agentManager.streamAgent(created.id, prompt);
+    await streamWithTimeout(iterator, opts.timeoutMs, async (event) => {
+      if (event.type === "permission_requested") {
+        await agentManager.respondToPermission(created.id, event.request.id, { behavior: "allow" });
+      }
+      if (event.type === "turn_failed") {
+        sawTurnFailed = true;
+        failureError = event.error;
+      }
+      if (event.type === "timeline" && event.item.type === "tool_call") {
+        logger.info({ name: event.item.name, callId: event.item.callId }, "Timeline tool call");
+        if (typeof event.item.name === "string" && event.item.name.toLowerCase().includes("speak")) {
+          sawSpeakTimeline = true;
+        }
+      }
+    });
+
+    console.log(
+      JSON.stringify(
+        {
+          ok: !sawTurnFailed && sawSpeakTimeline && speakCalls.length > 0,
+          provider: opts.provider,
+          agentId,
+          sawSpeakTimeline,
+          speakCalls,
+          sawTurnFailed,
+          failureError,
+        },
+        null,
+        2
+      )
+    );
+  } finally {
+    await agentManager.closeAgent(agentId).catch(() => undefined);
+    await agentManager.flush().catch(() => undefined);
+    await voiceBridgeManager.stop().catch(() => undefined);
+    await shutdownProviders(logger).catch(() => undefined);
+    await rm(tmpRoot, { recursive: true, force: true });
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});
@@ -0,0 +1,164 @@
+import { randomUUID } from "node:crypto";
+import pino from "pino";
+
+import { createTestPaseoDaemon } from "../src/server/test-utils/paseo-daemon.js";
+import { DaemonClient } from "../src/server/test-utils/daemon-client.js";
+import { OpenAITTS } from "../src/server/speech/providers/openai/tts.js";
+
+async function streamToBuffer(stream: AsyncIterable<unknown>): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as ArrayBufferLike));
+  }
+  return Buffer.concat(chunks);
+}
+
+async function main(): Promise<void> {
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) {
+    throw new Error("OPENAI_API_KEY is required");
+  }
+
+  const logger = pino({ level: process.env.PASEO_LOG_LEVEL ?? "info" });
+  const daemon = await createTestPaseoDaemon({
+    logger,
+    agentClients: {},
+    openai: { apiKey },
+    speech: {
+      providers: {
+        dictationStt: { provider: "openai", explicit: true },
+        voiceStt: { provider: "openai", explicit: true },
+        voiceTts: { provider: "openai", explicit: true },
+      },
+    },
+    voiceLlmProvider: "claude",
+    voiceLlmProviderExplicit: true,
+  });
+
+  const client = new DaemonClient({
+    url: `ws://127.0.0.1:${daemon.port}/ws`,
+  });
+
+  const cleanup = async () => {
+    await client.close().catch(() => undefined);
+    await daemon.close().catch(() => undefined);
+  };
+
+  try {
+    await client.connect();
+    client.subscribeAgentUpdates({ subscriptionId: "voice-debug" });
+
+    const voiceAgentId = randomUUID();
+    const mode = await client.setVoiceMode(true, voiceAgentId);
+    console.log("set_voice_mode_response", mode);
+
+    const offTranscript = client.on("transcription_result", (msg) => {
+      if (msg.type !== "transcription_result") return;
+      console.log("transcription_result", {
+        text: msg.payload.text,
+        isLowConfidence: msg.payload.isLowConfidence,
+      });
+    });
+
+    const offActivity = client.on("activity_log", (msg) => {
+      if (msg.type !== "activity_log") return;
+      if (msg.payload.type === "transcript" || msg.payload.type === "error" || msg.payload.type === "assistant") {
+        console.log("activity_log", {
+          type: msg.payload.type,
+          content: msg.payload.content,
+        });
+      }
+    });
+
+    const offStream = client.on("agent_stream", (msg) => {
+      if (msg.type !== "agent_stream") return;
+      if (msg.payload.event.type !== "timeline") return;
+      const item = msg.payload.event.item;
+      if (item.type !== "tool_call") return;
+      console.log("agent_stream:tool_call", {
+        agentId: msg.payload.agentId,
+        name: item.name,
+        status: item.status,
+      });
+    });
+
+    let audioChunkCount = 0;
+    const firstAudio = new Promise<void>((resolve) => {
+      const offAudio = client.on("audio_output", (msg) => {
+        if (msg.type !== "audio_output") return;
+        audioChunkCount += 1;
+        console.log("audio_output", {
+          id: msg.payload.id,
+          groupId: msg.payload.groupId,
+          chunkIndex: msg.payload.chunkIndex,
+          isLastChunk: msg.payload.isLastChunk,
+          format: msg.payload.format,
+        });
+        if (audioChunkCount === 1) {
+          offAudio();
+          resolve();
+        }
+      });
+    });
+    const lastAudio = new Promise<void>((resolve) => {
+      const offAudio = client.on("audio_output", (msg) => {
+        if (msg.type !== "audio_output") return;
+        if (msg.payload.isLastChunk) {
+          offAudio();
+          resolve();
+        }
+      });
+    });
+
+    const tts = new OpenAITTS(
+      {
+        apiKey,
+        responseFormat: "pcm",
+        voice: "alloy",
+      },
+      logger
+    );
+    const generated = await tts.synthesizeSpeech(
+      "Use the speak tool and say exactly round trip successful."
+    );
+    const pcm = await streamToBuffer(generated.stream as AsyncIterable<unknown>);
+
+    const chunkBytes = 4800;
+    for (let offset = 0; offset < pcm.length; offset += chunkBytes) {
+      const chunk = pcm.subarray(offset, Math.min(pcm.length, offset + chunkBytes));
+      const isLast = offset + chunkBytes >= pcm.length;
+      await client.sendVoiceAudioChunk(
+        chunk.toString("base64"),
+        "audio/pcm;rate=24000;bits=16",
+        isLast
+      );
+    }
+
+    await Promise.race([
+      firstAudio,
+      new Promise((_, reject) =>
+        setTimeout(() => reject(new Error("Timed out waiting for first audio_output")), 120000)
+      ),
+    ]);
+    await Promise.race([
+      lastAudio,
+      new Promise((_, reject) =>
+        setTimeout(() => reject(new Error("Timed out waiting for final audio_output")), 120000)
+      ),
+    ]);
+
+    console.log("success", { audioChunkCount });
+    await client.setVoiceMode(false);
+
+    offTranscript();
+    offActivity();
+    offStream();
+  } finally {
+    await cleanup();
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});