From a44038a0481dd98c1b20a9e83931423ecd43ca89 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 20:12:55 +0700 Subject: [PATCH] Update files --- .../src/server/agent/agent-mcp.e2e.test.ts | 1 - .../src/server/agent/provider-manifest.ts | 19 + .../server/src/server/bootstrap.smoke.test.ts | 1 - packages/server/src/server/bootstrap.ts | 81 ++- packages/server/src/server/config.ts | 20 +- .../src/server/daemon-client.e2e.test.ts | 24 +- .../server/src/server/persisted-config.ts | 6 +- packages/server/src/server/session.ts | 582 +----------------- .../src/server/test-utils/paseo-daemon.ts | 2 - .../server/src/server/websocket-server.ts | 13 +- .../public/schemas/paseo.config.v1.json | 53 +- .../website/src/routes/docs/configuration.tsx | 14 +- 12 files changed, 175 insertions(+), 641 deletions(-) diff --git a/packages/server/src/server/agent/agent-mcp.e2e.test.ts b/packages/server/src/server/agent/agent-mcp.e2e.test.ts index 527725f9f..d03485365 100644 --- a/packages/server/src/server/agent/agent-mcp.e2e.test.ts +++ b/packages/server/src/server/agent/agent-mcp.e2e.test.ts @@ -90,7 +90,6 @@ describe("agent MCP end-to-end (offline)", () => { mcpDebug: false, agentClients: createTestAgentClients(), agentStoragePath: path.join(paseoHome, "agents"), - openrouterApiKey: null, }; const daemon = await createPaseoDaemon(daemonConfig, pino({ level: "silent" })); diff --git a/packages/server/src/server/agent/provider-manifest.ts b/packages/server/src/server/agent/provider-manifest.ts index 9f20d861f..189ac78f7 100644 --- a/packages/server/src/server/agent/provider-manifest.ts +++ b/packages/server/src/server/agent/provider-manifest.ts @@ -7,6 +7,11 @@ export interface AgentProviderDefinition { description: string; defaultModeId: string | null; modes: AgentMode[]; + voice?: { + enabled: boolean; + defaultModeId: string; + defaultModel?: string; + }; } const CLAUDE_MODES: AgentMode[] = [ @@ -68,6 +73,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [ "Anthropic's multi-tool assistant with MCP support, streaming, and deep reasoning", defaultModeId: "default", modes: CLAUDE_MODES, + voice: { + enabled: true, + defaultModeId: "default", + defaultModel: "haiku", + }, }, { id: "codex", @@ -76,6 +86,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [ "OpenAI's Codex workspace agent with sandbox controls and optional network access", defaultModeId: "auto", modes: CODEX_MODES, + voice: { + enabled: true, + defaultModeId: "read-only", + defaultModel: "gpt-5.2-mini", + }, }, { id: "opencode", @@ -84,6 +99,10 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [ "Open-source coding assistant with multi-provider model support", defaultModeId: "default", modes: OPENCODE_MODES, + voice: { + enabled: true, + defaultModeId: "default", + }, }, ]; diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 8ee4a3d14..82e82984a 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -60,7 +60,6 @@ describe("paseo daemon bootstrap", () => { voiceSttProvider: "openai", voiceTtsProvider: "openai", }, - openrouterApiKey: null, }; try { diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index d88b9c3b1..4e62e14c3 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -74,13 +74,12 @@ import type { AgentClient, AgentProvider, } from "./agent/agent-sdk-types.js"; +import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js"; import { acquirePidLock, releasePidLock } from "./pid-lock.js"; import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js"; import { createVoiceMcpBridgeSocketServer, type VoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js"; type AgentMcpTransportMap = Map; -type VoiceAgentProvider = "claude" | "codex" | "opencode"; -const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"]; function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } { const explicit = process.env.PASEO_BIN_PATH?.trim(); @@ -147,8 +146,7 @@ export type PaseoDaemonConfig = { appBaseUrl?: string; openai?: PaseoOpenAIConfig; speech?: PaseoSpeechConfig; - openrouterApiKey?: string | null; - voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null; + voiceLlmProvider?: AgentProvider | null; voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; dictationFinalTimeoutMs?: number; @@ -313,21 +311,24 @@ export async function createPaseoDaemon( const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null; const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false; + const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS + .filter((definition) => definition.voice?.enabled) + .map((definition) => definition.id as AgentProvider); logger.info( { requestedVoiceLlmProvider, voiceLlmProviderExplicit, + voiceEnabledProviders, }, "Voice LLM provider reconciliation started" ); const providerClients = createAllClients(logger); - const voiceLlmAvailability: Record = { - claude: false, - codex: false, - opencode: false, - }; - for (const provider of VOICE_AGENT_FALLBACK_ORDER) { + Object.assign(providerClients, config.agentClients); + const voiceLlmAvailability = Object.fromEntries( + voiceEnabledProviders.map((provider) => [provider, false]) + ) as Record; + for (const provider of voiceEnabledProviders) { try { voiceLlmAvailability[provider] = await providerClients[provider].isAvailable(); } catch (error) { @@ -336,21 +337,17 @@ export async function createPaseoDaemon( } } - const voiceLlmDefaultProvider = - VOICE_AGENT_FALLBACK_ORDER.find((provider) => voiceLlmAvailability[provider]) ?? null; - - if (requestedVoiceLlmProvider === "openrouter") { - const openrouterApiKey = - config.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null; - if (!openrouterApiKey) { - logger.error("voiceMode.llm.provider is openrouter but no OpenRouter API key is configured"); - throw new Error("Missing OpenRouter API key for voiceMode.llm.provider=openrouter"); + let resolvedVoiceLlmProvider: AgentProvider | null = null; + if (requestedVoiceLlmProvider) { + if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) { + logger.error( + { provider: requestedVoiceLlmProvider, voiceEnabledProviders }, + "Configured voice LLM provider does not support voice mode" + ); + throw new Error( + `Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode` + ); } - } else if ( - requestedVoiceLlmProvider === "claude" || - requestedVoiceLlmProvider === "codex" || - requestedVoiceLlmProvider === "opencode" - ) { if (!voiceLlmAvailability[requestedVoiceLlmProvider]) { logger.error( { provider: requestedVoiceLlmProvider, voiceLlmAvailability }, @@ -358,20 +355,40 @@ export async function createPaseoDaemon( ); throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`); } - } else if (!voiceLlmDefaultProvider) { + resolvedVoiceLlmProvider = requestedVoiceLlmProvider; + } else { + resolvedVoiceLlmProvider = + voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null; + } + + if (!resolvedVoiceLlmProvider) { logger.error( { requestedVoiceLlmProvider, voiceLlmAvailability }, - "No local voice LLM provider available for fallback" + "No voice LLM provider available" ); - throw new Error("No local voice LLM provider available (claude/codex/opencode)"); + throw new Error("No voice LLM provider available"); } + const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find( + (definition) => definition.id === resolvedVoiceLlmProvider + ); + if (!resolvedVoiceProviderDefinition?.voice?.enabled) { + throw new Error( + `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry` + ); + } + const resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId; + const resolvedVoiceLlmModel = + config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null; + logger.info( { requestedVoiceLlmProvider, voiceLlmProviderExplicit, + resolvedVoiceLlmProvider, + resolvedVoiceLlmModeId, + resolvedVoiceLlmModel, voiceLlmAvailability, - voiceLlmDefaultProvider, }, "Voice LLM provider reconciliation completed" ); @@ -926,12 +943,10 @@ export async function createPaseoDaemon( { stt: sttService, tts: ttsService }, terminalManager, { - openrouterApiKey: config.openrouterApiKey ?? null, - voiceLlmProvider: config.voiceLlmProvider ?? null, + voiceLlmProvider: resolvedVoiceLlmProvider, + voiceLlmModeId: resolvedVoiceLlmModeId, voiceLlmProviderExplicit, - voiceLlmDefaultProvider, - voiceLlmModel: config.voiceLlmModel ?? null, - voiceLlmAvailability, + voiceLlmModel: resolvedVoiceLlmModel, voiceAgentMcpStdio: { command: voiceMcpBridgeCommand.command, baseArgs: [ diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index 47c1cd6f8..bbd455bdd 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -4,6 +4,8 @@ import type { PaseoDaemonConfig } from "./bootstrap.js"; import type { STTConfig } from "./speech/providers/openai/stt.js"; import type { TTSConfig } from "./speech/providers/openai/tts.js"; import { loadPersistedConfig } from "./persisted-config.js"; +import type { AgentProvider } from "./agent/agent-sdk-types.js"; +import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; import { mergeAllowedHosts, parseAllowedHostsEnv, @@ -13,15 +15,6 @@ import { const DEFAULT_PORT = 6767; const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443"; const DEFAULT_APP_BASE_URL = "https://app.paseo.sh"; -const VOICE_LLM_PROVIDER_IDS = [ - "openrouter", - "local-agent", - "claude", - "codex", - "opencode", -] as const; -type VoiceLlmProviderId = (typeof VOICE_LLM_PROVIDER_IDS)[number]; - function getDefaultListen(): string { // Main HTTP server defaults to TCP return `127.0.0.1:${DEFAULT_PORT}`; @@ -97,7 +90,7 @@ function parseSpeechProviderId(value: unknown): "openai" | "local" | null { return null; } -function parseVoiceLlmProviderId(value: unknown): VoiceLlmProviderId | null { +function parseVoiceLlmProviderId(value: unknown): AgentProvider | null { if (typeof value !== "string") { return null; } @@ -105,8 +98,8 @@ function parseVoiceLlmProviderId(value: unknown): VoiceLlmProviderId | null { if (!normalized) { return null; } - return (VOICE_LLM_PROVIDER_IDS as readonly string[]).includes(normalized) - ? (normalized as VoiceLlmProviderId) + return (AGENT_PROVIDER_IDS as readonly string[]).includes(normalized) + ? (normalized as AgentProvider) : null; } @@ -265,8 +258,6 @@ export function loadConfig( } : undefined; - const openrouterApiKey = - env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null; const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER); const persistedVoiceLlmProvider = parseVoiceLlmProviderId( persisted.features?.voiceMode?.llm?.provider @@ -299,7 +290,6 @@ export function loadConfig( voiceTtsProvider, ...(sherpaOnnx ? { sherpaOnnx } : {}), }, - openrouterApiKey, voiceLlmProvider, voiceLlmProviderExplicit, voiceLlmModel, diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index 06b0f7466..b395c1191 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -574,16 +574,16 @@ describe("daemon client E2E", () => { 120000 ); - test.runIf(Boolean(process.env.OPENROUTER_API_KEY))( - "streams session activity logs and chunks", + test( + "does not process non-voice LLM turns via OpenRouter", async () => { await ctx.client.setVoiceConversation(false); - let sawAssistantChunk = false; let sawTranscriptLog = false; + let sawAssistantChunk = false; let sawAssistantLog = false; - const completion = waitForSignal(60000, (resolve) => { + const transcriptSeen = waitForSignal(60000, (resolve) => { const unsubscribeChunk = ctx.client.on("assistant_chunk", (message) => { if (message.type !== "assistant_chunk") { return; @@ -591,9 +591,6 @@ describe("daemon client E2E", () => { if (message.payload.chunk.length > 0) { sawAssistantChunk = true; } - if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) { - resolve(); - } }); const unsubscribeActivity = ctx.client.on("activity_log", (message) => { @@ -602,13 +599,11 @@ describe("daemon client E2E", () => { } if (message.payload.type === "transcript") { sawTranscriptLog = true; + resolve(); } if (message.payload.type === "assistant") { sawAssistantLog = true; } - if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) { - resolve(); - } }); return () => { @@ -618,9 +613,14 @@ describe("daemon client E2E", () => { }); await ctx.client.sendUserMessage("Say 'hello' and nothing else"); - await completion; + await transcriptSeen; + await new Promise((resolve) => setTimeout(resolve, 1500)); + + expect(sawTranscriptLog).toBe(true); + expect(sawAssistantChunk).toBe(false); + expect(sawAssistantLog).toBe(false); }, - 120000 + 90000 ); speechTest( diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index be1293241..2ee622536 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { z } from "zod"; +import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; const LogConfigSchema = z .object({ @@ -41,7 +42,6 @@ const SherpaOnnxProviderSchema = z const ProvidersSchema = z .object({ openai: ProviderCredentialsSchema.optional(), - openrouter: ProviderCredentialsSchema.optional(), sherpaOnnx: SherpaOnnxProviderSchema.optional(), }) .strict(); @@ -74,9 +74,7 @@ const FeatureVoiceModeSchema = z .object({ llm: z .object({ - provider: z - .enum(["openrouter", "local-agent", "claude", "codex", "opencode"]) - .optional(), + provider: z.enum(AGENT_PROVIDER_IDS as [string, ...string[]]).optional(), model: z.string().min(1).optional(), }) .strict() diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index d03196d6e..7249e4d9e 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1,17 +1,10 @@ import { v4 as uuidv4 } from "uuid"; -import { readFile, mkdir, writeFile, stat } from "fs/promises"; +import { mkdir, stat } from "fs/promises"; import { exec } from "child_process"; -import { promisify, inspect } from "util"; +import { promisify } from "util"; import { join, resolve, sep } from "path"; -import invariant from "tiny-invariant"; import { z } from "zod"; -import { streamText, stepCountIs } from "ai"; import type { ToolSet } from "ai"; -import type { ModelMessage } from "@ai-sdk/provider-utils"; -import { - createOpenRouter, - OpenRouterProviderOptions, -} from "@openrouter/ai-sdk-provider"; import { serializeAgentStreamEvent, type AgentSnapshotPayload, @@ -29,8 +22,6 @@ import { } from "./messages.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import { parseAndHighlightDiff, type ParsedDiffFile } from "./utils/diff-highlighter.js"; -import { getSystemPrompt } from "./agent/system-prompt.js"; -import { getAllTools } from "./agent/llm-openai.js"; import { TTSManager } from "./agent/tts-manager.js"; import { STTManager } from "./agent/stt-manager.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; @@ -48,8 +39,6 @@ import { experimental_createMCPClient } from "ai"; import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js"; export type AgentMcpTransportFactory = () => Promise; -type VoiceLlmProvider = "openrouter" | "local-agent" | "claude" | "codex" | "opencode"; -type VoiceAgentProvider = Exclude; type VoiceSpeakHandler = (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise; type VoiceCallerContext = { childAgentDefaultLabels?: Record; @@ -144,16 +133,6 @@ const RESTART_EXIT_DELAY_MS = 250; * Uses Claude Haiku for speed and cost efficiency. */ const AUTO_GEN_MODEL = "haiku"; -const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"]; -const VOICE_AGENT_DEFAULT_MODE: Record = { - claude: "default", - codex: "read-only", - opencode: "default", -}; -const VOICE_AGENT_DEFAULT_MODEL: Partial> = { - claude: "haiku", - codex: "gpt-5.2-mini", -}; const VOICE_AGENT_SYSTEM_INSTRUCTION = [ "You are the Paseo voice assistant.", "The user cannot see your chat messages or tool calls.", @@ -210,18 +189,6 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round( ); const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/; -/** - * Type for present_artifact tool arguments - */ -interface PresentArtifactArgs { - type: "markdown" | "diff" | "image" | "code"; - source: - | { type: "file"; path: string } - | { type: "command_output"; command: string } - | { type: "text"; text: string }; - title: string; -} - interface AudioBufferState { chunks: Buffer[]; format: string; @@ -335,7 +302,6 @@ export class Session { >(); // Conversation history - private messages: ModelMessage[] = []; private turnIndex = 0; // Per-session managers @@ -368,12 +334,10 @@ export class Session { } | null = null; private readonly terminalManager: TerminalManager | null; private terminalSubscriptions: Map void> = new Map(); - private readonly openrouterApiKey: string | null; - private readonly voiceLlmProvider: VoiceLlmProvider | null; + private readonly voiceLlmProvider: AgentProvider | null; + private readonly voiceLlmModeId: string | null; private readonly voiceLlmProviderExplicit: boolean; - private readonly voiceLlmDefaultProvider: VoiceAgentProvider | null; private readonly voiceLlmModel: string | null; - private readonly voiceLlmAvailability: Record | null; private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null; private readonly registerVoiceSpeakHandler?: ( agentId: string, @@ -401,12 +365,10 @@ export class Session { tts: TextToSpeechProvider | null, terminalManager: TerminalManager | null, voice?: { - openrouterApiKey?: string | null; - voiceLlmProvider?: VoiceLlmProvider | null; + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; voiceLlmProviderExplicit?: boolean; - voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; - voiceLlmAvailability?: Record | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; }, voiceBridge?: { @@ -430,12 +392,10 @@ export class Session { this.agentStorage = agentStorage; this.createAgentMcpTransport = createAgentMcpTransport; this.terminalManager = terminalManager; - this.openrouterApiKey = voice?.openrouterApiKey ?? null; this.voiceLlmProvider = voice?.voiceLlmProvider ?? null; + this.voiceLlmModeId = voice?.voiceLlmModeId ?? null; this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false; - this.voiceLlmDefaultProvider = voice?.voiceLlmDefaultProvider ?? null; this.voiceLlmModel = voice?.voiceLlmModel ?? null; - this.voiceLlmAvailability = voice?.voiceLlmAvailability ?? null; this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null; this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler; this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler; @@ -467,28 +427,6 @@ export class Session { this.sessionLogger.info("Session created"); } - private escapeXmlText(value: string): string { - return value - .replace(/&/g, "&") - .replace(//g, ">"); - } - - private escapeXmlAttribute(value: string): string { - return this.escapeXmlText(value) - .replace(/"/g, """) - .replace(/'/g, "'"); - } - - private formatVoiceTranscriptionForLLM(text: string): string { - const trimmed = text.trim(); - const focusedAgentId = this.clientActivity?.focusedAgentId ?? null; - const focusedAttr = focusedAgentId - ? ` focused-agent-id="${this.escapeXmlAttribute(focusedAgentId)}"` - : ""; - return `${this.escapeXmlText(trimmed)}`; - } - /** * Get the client's current activity state */ @@ -1195,19 +1133,6 @@ export class Session { voiceConversationId: string, requestId: string ): Promise { - if (this.voiceLlmProvider === "openrouter") { - this.voiceAssistantAgentId = null; - this.messages = []; - this.emit({ - type: "voice_conversation_loaded", - payload: { - voiceConversationId, - messageCount: 0, - requestId, - }, - }); - return; - } this.voiceAssistantAgentId = voiceConversationId; this.emit({ @@ -1459,36 +1384,19 @@ export class Session { } this.isVoiceMode = true; - if (this.voiceLlmProvider !== "openrouter") { - this.voiceAssistantAgentId = voiceConversationId; - this.sessionLogger.info( - { voiceAssistantAgentId: this.voiceAssistantAgentId }, - "Voice conversation enabled (agent-backed)" - ); - return; - } - - // OpenRouter voice mode is always ephemeral. - this.voiceAssistantAgentId = null; - this.messages = []; - + this.voiceAssistantAgentId = voiceConversationId; this.sessionLogger.info( - { messageCount: this.messages.length }, - "Voice conversation enabled" + { voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Voice conversation enabled (agent-backed)" ); return; } this.isVoiceMode = false; - if (this.voiceLlmProvider !== "openrouter") { - this.sessionLogger.info( - { voiceAssistantAgentId: this.voiceAssistantAgentId }, - "Voice conversation disabled (agent-backed)" - ); - return; - } - this.voiceAssistantAgentId = null; - this.sessionLogger.info("Voice conversation disabled"); + this.sessionLogger.info( + { voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Voice conversation disabled (agent-backed)" + ); } /** @@ -4342,10 +4250,7 @@ export class Session { }, }); - // Add to conversation - this.messages.push({ role: "user", content: text }); - - // Process through LLM (TTS enabled in voice mode for voice conversations) + // Process through LLM (voice path is agent-backed only) this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, text); await this.currentStreamPromise; } @@ -4612,14 +4517,6 @@ export class Session { }, }); - // Add to conversation - this.messages.push({ - role: "user", - content: this.isVoiceMode - ? this.formatVoiceTranscriptionForLLM(result.text) - : result.text, - }); - // Set phase to LLM and process (TTS enabled in voice mode for voice conversations) this.clearSpeechInProgress("transcription complete"); this.setPhase("llm"); @@ -4642,52 +4539,6 @@ export class Session { } } - /** - * Resolve the effective voice LLM provider. - * - explicit provider => strict - * - local-agent / unset => fallback order - */ - private resolveVoiceAgentProvider(): VoiceAgentProvider { - const configured = this.voiceLlmProvider; - const availability = this.voiceLlmAvailability ?? { - claude: true, - codex: true, - opencode: true, - }; - - if (configured === "openrouter") { - throw new Error("voiceLlmProvider=openrouter cannot be used in local-agent flow"); - } - - if (configured === "claude" || configured === "codex" || configured === "opencode") { - if (!availability[configured]) { - throw new Error(`Configured voice LLM provider '${configured}' is unavailable`); - } - return configured; - } - - const fallbackOrder = - this.voiceLlmDefaultProvider && availability[this.voiceLlmDefaultProvider] - ? [this.voiceLlmDefaultProvider, ...VOICE_AGENT_FALLBACK_ORDER.filter((id) => id !== this.voiceLlmDefaultProvider)] - : VOICE_AGENT_FALLBACK_ORDER; - - for (const provider of fallbackOrder) { - if (availability[provider]) { - return provider; - } - } - - throw new Error("No local voice LLM provider is available (claude/codex/opencode)"); - } - - private getVoiceAgentModel(provider: VoiceAgentProvider): string | undefined { - const configured = this.voiceLlmModel?.trim(); - if (configured) { - return configured; - } - return VOICE_AGENT_DEFAULT_MODEL[provider]; - } - private async ensureVoiceAssistantAgent(): Promise { if (this.voiceAssistantAgentId) { const existing = this.agentManager.getAgent(this.voiceAssistantAgentId); @@ -4706,7 +4557,10 @@ export class Session { } } - const provider = this.resolveVoiceAgentProvider(); + const provider = this.voiceLlmProvider; + if (!provider) { + throw new Error("Voice LLM provider is not configured"); + } const voiceAgentId = this.voiceAssistantAgentId ?? uuidv4(); const cwd = join(this.paseoHome, "voice-agent-workspace"); await mkdir(cwd, { recursive: true }); @@ -4716,11 +4570,11 @@ export class Session { throw new Error("Voice MCP stdio bridge is not configured"); } - const model = this.getVoiceAgentModel(provider); + const model = this.voiceLlmModel?.trim() || undefined; const config: AgentSessionConfig = { provider, cwd, - modeId: VOICE_AGENT_DEFAULT_MODE[provider], + modeId: this.voiceLlmModeId ?? "default", ...(model ? { model } : {}), mcpServers: { paseo: buildVoiceAgentMcpServerConfig({ @@ -4872,372 +4726,22 @@ export class Session { * Process user message through LLM with streaming and tool execution */ private async processWithLLM(enableTTS: boolean, latestUserText?: string): Promise { - if (enableTTS && this.voiceLlmProvider !== "openrouter") { - const text = - typeof latestUserText === "string" && latestUserText.trim().length > 0 - ? latestUserText - : (() => { - const lastUser = [...this.messages] - .reverse() - .find((message) => message.role === "user"); - if (!lastUser) { - return ""; - } - return typeof lastUser.content === "string" - ? lastUser.content - : JSON.stringify(lastUser.content); - })(); - const normalized = text.trim(); + try { + if (!enableTTS) { + this.sessionLogger.warn("Ignoring non-voice processWithLLM call; voice is agent-only"); + return; + } + const normalized = (latestUserText ?? "").trim(); if (!normalized) { return; } await this.processWithVoiceAgent(normalized); - return; - } - - let assistantResponse = ""; - let pendingTTS: Promise | null = null; - let textBuffer = ""; - let sawTextDelta = false; - - const flushTextBuffer = () => { - if (textBuffer.length > 0) { - // TTS handling (capture mode at generation time for drift protection) - if (enableTTS && !this.speechInProgress) { - const modeAtGeneration = this.isVoiceMode; - pendingTTS = this.ttsManager.generateAndWaitForPlayback( - textBuffer, - (msg) => this.emit(msg), - this.abortController.signal, - modeAtGeneration - ); - } else if (enableTTS && this.speechInProgress) { - this.sessionLogger.debug("Skipping TTS chunk while speech in progress"); - } - - // Emit activity log - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "assistant", - content: textBuffer, - }, - }); - } - textBuffer = ""; - }; - - try { - // Debug: dump conversation before LLM call - await this.dumpConversation(); - - const openrouterApiKey = - this.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null; - invariant( - openrouterApiKey, - "OpenRouter API key is required (set providers.openrouter.apiKey in config.json or OPENROUTER_API_KEY)" - ); - - const openrouter = createOpenRouter({ - apiKey: openrouterApiKey, - }); - - // Wait for agent MCP to initialize if needed - if (!this.agentTools) { - this.sessionLogger.debug("Waiting for agent MCP initialization..."); - const startTime = Date.now(); - while (!this.agentTools && Date.now() - startTime < 5000) { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - if (!this.agentTools) { - this.sessionLogger.info("Agent MCP tools unavailable; continuing with default tool set"); - } - } - - const allTools = getAllTools(this.agentTools ?? undefined); - - const result = await streamText({ - model: openrouter(this.voiceLlmModel ?? "anthropic/claude-haiku-4.5"), - system: getSystemPrompt(), - providerOptions: { - openrouter: { - transforms: ["middle-out"], // Compress prompts that are > context size. - } as OpenRouterProviderOptions, - }, - messages: this.messages, - tools: allTools, - abortSignal: this.abortController.signal, - onFinish: async (event) => { - const newMessages = event.response.messages; - if (newMessages.length > 0) { - this.messages.push(...newMessages); - this.sessionLogger.debug( - { messageCount: newMessages.length }, - `onFinish - saved message with ${newMessages.length} steps` - ); - } - }, - onChunk: async ({ chunk }) => { - if (chunk.type === "text-delta") { - sawTextDelta = true; - // Accumulate text in buffer - textBuffer += chunk.text; - assistantResponse += chunk.text; - - // Emit chunk for UI streaming - this.emit({ - type: "assistant_chunk", - payload: { chunk: chunk.text }, - }); - } else if (chunk.type === "tool-call") { - // Flush accumulated text as a segment before tool call - flushTextBuffer(); - - // Wait for pending TTS before executing tool - if (pendingTTS) { - this.sessionLogger.debug( - { toolName: chunk.toolName }, - `Waiting for TTS before executing ${chunk.toolName}` - ); - await pendingTTS; - } - - // Handle present_artifact tool specially - if (chunk.toolName === "present_artifact") { - await this.handlePresentArtifact( - chunk.toolCallId, - chunk.input as PresentArtifactArgs - ); - } - - // Emit tool call activity log - this.emit({ - type: "activity_log", - payload: { - id: chunk.toolCallId, - timestamp: new Date(), - type: "tool_call", - content: `Calling ${chunk.toolName}`, - metadata: { - toolCallId: chunk.toolCallId, - toolName: chunk.toolName, - arguments: chunk.input, - }, - }, - }); - } else if (chunk.type === "tool-result") { - // Check if this is a create_agent result - if (chunk.toolName === "create_agent" && chunk.output) { - const result = chunk.output as any; - if (result.structuredContent?.agentId) { - const agentId = result.structuredContent.agentId; - this.emit({ - type: "status", - payload: { - status: "agent_created", - agentId, - }, - }); - } - } - - // Emit tool result event - this.emit({ - type: "activity_log", - payload: { - id: chunk.toolCallId, - timestamp: new Date(), - type: "tool_result", - content: `Tool ${chunk.toolName} completed`, - metadata: { - toolCallId: chunk.toolCallId, - toolName: chunk.toolName, - result: chunk.output, - }, - }, - }); - } - }, - onError: async (error) => { - this.sessionLogger.error({ err: error }, "Stream error"); - - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "error", - content: `Stream error: ${ - error instanceof Error ? error.message : String(error) - }`, - }, - }); - }, - stopWhen: stepCountIs(10), - }); - - // Consume the fullStream to handle tool-error chunks - for await (const part of result.fullStream) { - if (part.type === "tool-error") { - this.emit({ - type: "activity_log", - payload: { - id: part.toolCallId, - timestamp: new Date(), - type: "error", - content: `Tool ${part.toolName} failed: ${ - part.error instanceof Error - ? part.error.message - : String(part.error) - }`, - metadata: { - toolCallId: part.toolCallId, - toolName: part.toolName, - error: part.error, - }, - }, - }); - } - } - - if (!sawTextDelta) { - let fallbackText = ""; - try { - fallbackText = (await result.text).trim(); - } catch { - fallbackText = ""; - } - if (fallbackText.length > 0) { - textBuffer += fallbackText; - assistantResponse += fallbackText; - this.emit({ - type: "assistant_chunk", - payload: { chunk: fallbackText }, - }); - } - } - - // Flush any remaining text at the end - flushTextBuffer(); - - // Note: Message is saved by onFinish callback with proper tool calls - - // Now wait for any pending TTS, but don't fail if it times out - if (pendingTTS) { - try { - await pendingTTS; - } catch (ttsError) { - this.sessionLogger.error( - { err: ttsError }, - "TTS playback failed (message already saved)" - ); - } - } - } catch (error) { - // Note: Partial messages are saved by onAbort callback with proper tool calls - - // Check if this is an abort error - const isAbortError = - error instanceof Error && error.name === "AbortError"; - - // Only emit error log for non-abort errors - if (!isAbortError) { - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "error", - content: `Error: ${ - error instanceof Error ? error.message : String(error) - }`, - }, - }); - } - - // Don't re-throw abort errors (they're expected during interruptions) - if (isAbortError) { - this.sessionLogger.debug("Stream aborted (partial response saved)"); - return; - } - - // Re-throw unexpected errors - throw error; } finally { - // Increment turn index for next LLM call this.turnIndex++; - - // Clear the stream promise tracker this.currentStreamPromise = null; } } - /** - * Handle present_artifact tool execution - */ - private async handlePresentArtifact( - toolCallId: string, - args: PresentArtifactArgs - ): Promise { - let content: string; - let isBase64 = false; - - try { - if (args.source.type === "file") { - const fileBuffer = await readFile(args.source.path); - content = fileBuffer.toString("base64"); - isBase64 = true; - } else if (args.source.type === "command_output") { - const { stdout } = await execAsync(args.source.command, { - encoding: "buffer", - }); - content = stdout.toString("base64"); - isBase64 = true; - } else if (args.source.type === "text") { - content = args.source.text; - isBase64 = false; - } else { - content = "[Unknown source type]"; - isBase64 = false; - } - } catch (error) { - this.sessionLogger.error( - { err: error }, - "Failed to resolve artifact source" - ); - content = `[Error: ${ - error instanceof Error ? error.message : String(error) - }]`; - isBase64 = false; - } - - // Emit artifact message - this.emit({ - type: "artifact", - payload: { - type: args.type, - id: toolCallId, - title: args.title, - content, - isBase64, - }, - }); - - // Emit activity log for artifact - this.emit({ - type: "activity_log", - payload: { - id: toolCallId, - timestamp: new Date(), - type: "system", - content: `${args.type} artifact: ${args.title}`, - metadata: { artifactId: toolCallId, artifactType: args.type }, - }, - }); - } - /** * Handle abort request from client */ @@ -5445,38 +4949,6 @@ export class Session { this.onMessage(msg); } - /** - * Debug helper: dump conversation to disk - */ - private async dumpConversation(): Promise { - try { - const dumpDir = join(process.cwd(), ".debug.conversations"); - await mkdir(dumpDir, { recursive: true }); - - const filename = `${this.sessionId}-${this.turnIndex}.json`; - const filepath = join(dumpDir, filename); - - const dump = { - voiceAssistantAgentId: this.voiceAssistantAgentId, - sessionId: this.sessionId, - turnIndex: this.turnIndex, - timestamp: new Date().toISOString(), - messages: this.messages, - }; - - await writeFile(filepath, inspect(dump, { depth: null }), "utf-8"); - this.sessionLogger.debug( - { filepath }, - `Dumped conversation to ${filepath}` - ); - } catch (error) { - this.sessionLogger.error( - { err: error }, - "Failed to dump conversation" - ); - } - } - /** * Clean up session resources */ diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index 74da63fab..44c1a69b8 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -21,7 +21,6 @@ type TestPaseoDaemonOptions = { cleanup?: boolean; openai?: PaseoOpenAIConfig; speech?: PaseoSpeechConfig; - openrouterApiKey?: string | null; voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"]; voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; @@ -82,7 +81,6 @@ export async function createTestPaseoDaemon( appBaseUrl: "https://app.paseo.sh", openai: options.openai, speech: options.speech, - openrouterApiKey: options.openrouterApiKey ?? null, voiceLlmProvider: options.voiceLlmProvider ?? null, voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false, voiceLlmModel: options.voiceLlmModel ?? null, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 11c500b5f..39b147368 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -22,7 +22,6 @@ import { PushService } from "./push/push-service.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; export type AgentMcpTransportFactory = () => Promise; -type VoiceAgentProvider = "claude" | "codex" | "opencode"; type VoiceMcpStdioConfig = { command: string; baseArgs: string[]; @@ -79,12 +78,10 @@ export class VoiceAssistantWebSocketServer { stt?: SpeechToTextProvider | null; } | null; private readonly voice: { - openrouterApiKey?: string | null; - voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null; + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; voiceLlmProviderExplicit?: boolean; - voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; - voiceLlmAvailability?: Record | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; } | null; private readonly voiceSpeakHandlers = new Map< @@ -114,12 +111,10 @@ export class VoiceAssistantWebSocketServer { speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null }, terminalManager?: TerminalManager | null, voice?: { - openrouterApiKey?: string | null; - voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null; + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; voiceLlmProviderExplicit?: boolean; - voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; - voiceLlmAvailability?: Record | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; }, dictation?: { diff --git a/packages/website/public/schemas/paseo.config.v1.json b/packages/website/public/schemas/paseo.config.v1.json index 14a45f390..116daee60 100644 --- a/packages/website/public/schemas/paseo.config.v1.json +++ b/packages/website/public/schemas/paseo.config.v1.json @@ -86,8 +86,44 @@ }, "additionalProperties": false }, - "openrouter": { - "$ref": "#/definitions/PaseoConfigV1/properties/providers/properties/openai" + "sherpaOnnx": { + "type": "object", + "properties": { + "modelsDir": { + "type": "string", + "minLength": 1 + }, + "autoDownload": { + "type": "boolean" + }, + "stt": { + "type": "object", + "properties": { + "preset": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "tts": { + "type": "object", + "properties": { + "preset": { + "type": "string", + "minLength": 1 + }, + "speakerId": { + "type": "number" + }, + "speed": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } }, "additionalProperties": false @@ -104,7 +140,8 @@ "provider": { "type": "string", "enum": [ - "openai" + "openai", + "local" ] }, "model": { @@ -129,7 +166,9 @@ "provider": { "type": "string", "enum": [ - "openrouter" + "claude", + "codex", + "opencode" ] }, "model": { @@ -145,7 +184,8 @@ "provider": { "type": "string", "enum": [ - "openai" + "openai", + "local" ] }, "model": { @@ -161,7 +201,8 @@ "provider": { "type": "string", "enum": [ - "openai" + "openai", + "local" ] }, "model": { diff --git a/packages/website/src/routes/docs/configuration.tsx b/packages/website/src/routes/docs/configuration.tsx index 96b671c60..169287660 100644 --- a/packages/website/src/routes/docs/configuration.tsx +++ b/packages/website/src/routes/docs/configuration.tsx @@ -67,8 +67,7 @@ function Configuration() { "$schema": "https://paseo.sh/schemas/paseo.config.v1.json", "version": 1, "providers": { - "openai": { "apiKey": "..." }, - "openrouter": { "apiKey": "..." } + "openai": { "apiKey": "..." } }, "daemon": { "listen": "127.0.0.1:6767", @@ -92,6 +91,7 @@ function Configuration() { "features": { "dictation": { "stt": { "provider": "local" } }, "voiceMode": { + "llm": { "provider": "claude", "model": "haiku" }, "stt": { "provider": "local" }, "tts": { "provider": "local" } } @@ -106,6 +106,13 @@ function Configuration() { } }`} +

+ Voice LLM orchestration is agents-only. Set{' '} + features.voiceMode.llm.provider to one of{' '} + claude, codex, or{' '} + opencode. If set, startup is strict and fails if unavailable. + If omitted, Paseo picks the first available voice-enabled agent provider. +

Local voice uses ONNX models (sherpa-onnx + PocketTTS). Default presets are parakeet-tdt-0.6b-v3-int8 for STT and @@ -152,7 +159,8 @@ function Configuration() {

  • PASEO_HOME — set Paseo home directory
  • PASEO_LISTEN — override daemon.listen
  • PASEO_ALLOWED_HOSTS — override/extend daemon.allowedHosts
    • -
  • OPENAI_API_KEY and OPENROUTER_API_KEY — override provider keys
    • +
  • OPENAI_API_KEY — override OpenAI provider key
    • +
  • PASEO_VOICE_LLM_PROVIDER — override voice LLM provider (claude, codex, opencode)
  • PASEO_DICTATION_STT_PROVIDER, PASEO_VOICE_STT_PROVIDER, PASEO_VOICE_TTS_PROVIDER — override voice provider selection (local or openai)
  • PASEO_SHERPA_ONNX_MODELS_DIR and PASEO_SHERPA_ONNX_AUTO_DOWNLOAD — control local model directory and download behavior
  • PASEO_SHERPA_STT_PRESET and PASEO_SHERPA_TTS_PRESET — override local STT/TTS model presets