From 3cc7feab216b5b9271ca654f39f76d3a66418e67 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sat, 7 Feb 2026 14:06:11 +0700 Subject: [PATCH] Update files --- .../app/src/components/agent-input-area.tsx | 80 +++- .../app/src/components/agent-stream-view.tsx | 3 +- packages/app/src/components/message-input.tsx | 15 +- .../app/src/components/sliding-sidebar.tsx | 200 +------- packages/app/src/contexts/voice-context.tsx | 37 +- .../server/scripts/voice-roundtrip-debug.ts | 18 +- packages/server/src/client/daemon-client.ts | 4 +- .../server/src/server/agent/agent-manager.ts | 33 +- .../server/src/server/agent/mcp-server.ts | 7 +- packages/server/src/server/bootstrap.ts | 99 +--- .../src/server/daemon-client.e2e.test.ts | 24 +- packages/server/src/server/session.ts | 436 +++++++++--------- .../local/sherpa/speech-download.e2e.test.ts | 16 +- .../src/server/voice-local-agent.e2e.test.ts | 142 +++--- .../server/voice-permission-policy.test.ts | 10 +- .../src/server/voice-permission-policy.ts | 12 +- .../src/server/voice-roundtrip.e2e.test.ts | 38 +- .../server/src/server/websocket-server.ts | 8 - packages/server/src/shared/messages.ts | 4 +- 19 files changed, 556 insertions(+), 630 deletions(-) diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx index f1b062432..b66b2acd9 100644 --- a/packages/app/src/components/agent-input-area.tsx +++ b/packages/app/src/components/agent-input-area.tsx @@ -4,10 +4,11 @@ import { Text, ActivityIndicator, Platform, + Alert, } from "react-native"; import { useState, useEffect, useRef, useCallback, useMemo } from "react"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; -import { ArrowUp, Square, Pencil } from "lucide-react-native"; +import { ArrowUp, Square, Pencil, AudioLines } from "lucide-react-native"; import Animated, { useAnimatedStyle } from "react-native-reanimated"; import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; import { useSafeAreaInsets } from "react-native-safe-area-context"; @@ -30,6 +31,7 @@ import { useAgentCommandsQuery } from "@/hooks/use-agent-commands-query"; import { encodeImages } from "@/utils/encode-images"; import { useKeyboardNavStore } from "@/stores/keyboard-nav-store"; import { focusWithRetries } from "@/utils/web-focus"; +import { useVoiceOptional } from "@/contexts/voice-context"; type QueuedMessage = { id: string; @@ -76,6 +78,7 @@ export function AgentInputArea({ const client = useSessionStore( (state) => state.sessions[serverId]?.client ?? null ); + const voice = useVoiceOptional(); const isConnected = client?.isConnected ?? false; const agent = useSessionStore((state) => @@ -411,6 +414,29 @@ export function AgentInputArea({ messageInputRef.current?.focus(); } + const isVoiceModeForAgent = + voice?.isVoiceModeForAgent(serverId, agentId) ?? false; + + const handleToggleRealtimeVoice = useCallback(() => { + if (!voice || !isConnected) { + return; + } + if (voice.isVoiceSwitching) { + return; + } + if (voice.isVoiceModeForAgent(serverId, agentId)) { + void voice.stopVoice().catch((error) => { + console.error("[AgentInputArea] Failed to stop voice mode", error); + Alert.alert("Voice failed", "Unable to stop realtime voice mode."); + }); + return; + } + void voice.startVoice(serverId, agentId).catch((error) => { + console.error("[AgentInputArea] Failed to start voice mode", error); + Alert.alert("Voice failed", "Unable to start realtime voice mode."); + }); + }, [agentId, isConnected, serverId, voice]); + function handleEditQueuedMessage(id: string) { const item = queuedMessages.find((q) => q.id === id); if (!item) return; @@ -494,7 +520,7 @@ export function AgentInputArea({ ] ); - const rightContent = isAgentRunning ? ( + const cancelButton = isAgentRunning ? ( ) : null; + const rightContent = ( + + + {voice?.isVoiceSwitching ? ( + + ) : isVoiceModeForAgent ? ( + + ) : ( + + )} + + {cancelButton} + + ); + const leftContent = ; return ( @@ -588,6 +643,8 @@ export function AgentInputArea({ isScreenFocused={isScreenFocused} leftContent={leftContent} rightContent={rightContent} + voiceServerId={serverId} + voiceAgentId={agentId} isAgentRunning={isAgentRunning} onQueue={handleQueue} onKeyPress={handleCommandKeyPress} @@ -631,6 +688,25 @@ const styles = StyleSheet.create(((theme: Theme) => ({ alignItems: "center", justifyContent: "center", }, + rightControls: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + realtimeVoiceButton: { + width: 34, + height: 34, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.surface0, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + alignItems: "center", + justifyContent: "center", + }, + realtimeVoiceButtonActive: { + backgroundColor: theme.colors.palette.green[600], + borderColor: theme.colors.palette.green[800], + }, buttonDisabled: { opacity: 0.5, }, diff --git a/packages/app/src/components/agent-stream-view.tsx b/packages/app/src/components/agent-stream-view.tsx index 379f8a5f4..458f48b22 100644 --- a/packages/app/src/components/agent-stream-view.tsx +++ b/packages/app/src/components/agent-stream-view.tsx @@ -102,7 +102,7 @@ export function AgentStreamView({ const flatListRef = useRef>(null); const { theme } = useUnistyles(); const insets = useSafeAreaInsets(); - const { isVoiceMode } = useVoice(); + const { isVoiceModeForAgent } = useVoice(); const [isNearBottom, setIsNearBottom] = useState(true); const hasScrolledInitially = useRef(false); const hasAutoScrolledOnce = useRef(false); @@ -114,6 +114,7 @@ export function AgentStreamView({ // Get serverId (fallback to agent's serverId if not provided) const resolvedServerId = serverId ?? agent.serverId ?? ""; + const isVoiceMode = isVoiceModeForAgent(resolvedServerId, agentId); const client = useSessionStore( (state) => state.sessions[resolvedServerId]?.client ?? null diff --git a/packages/app/src/components/message-input.tsx b/packages/app/src/components/message-input.tsx index 09ae2721a..60db3b1de 100644 --- a/packages/app/src/components/message-input.tsx +++ b/packages/app/src/components/message-input.tsx @@ -61,6 +61,8 @@ export interface MessageInputProps { leftContent?: React.ReactNode; /** Content to render on the right side after voice button (e.g., realtime button, cancel button) */ rightContent?: React.ReactNode; + voiceServerId?: string; + voiceAgentId?: string; /** When true and there's sendable content, calls onQueue instead of onSubmit */ isAgentRunning?: boolean; /** Callback for queue button when agent is running */ @@ -117,6 +119,8 @@ export const MessageInput = forwardRef( isScreenFocused = true, leftContent, rightContent, + voiceServerId, + voiceAgentId, isAgentRunning = false, onQueue, onKeyPress: onKeyPressCallback, @@ -310,15 +314,20 @@ export const MessageInput = forwardRef( })); const handleVoicePress = useCallback(async () => { + const isRealtimeVoiceForCurrentAgent = + voice && + !!voiceServerId && + !!voiceAgentId && + voice.isVoiceModeForAgent(voiceServerId, voiceAgentId); if (isDictating) { await cancelDictation(); } else { - if (voice?.isVoiceMode) { + if (isRealtimeVoiceForCurrentAgent) { await voice.stopVoice(); } await startDictation(); } - }, [isDictating, cancelDictation, startDictation, voice]); + }, [isDictating, cancelDictation, startDictation, voice, voiceAgentId, voiceServerId]); const handleCancelRecording = useCallback(async () => { await cancelDictation(); @@ -626,6 +635,7 @@ export const MessageInput = forwardRef( )} + {rightContent} {shouldShowSendButton && ( ( )} - {!shouldShowSendButton && rightContent} diff --git a/packages/app/src/components/sliding-sidebar.tsx b/packages/app/src/components/sliding-sidebar.tsx index 05b12aeb9..c3fb67de2 100644 --- a/packages/app/src/components/sliding-sidebar.tsx +++ b/packages/app/src/components/sliding-sidebar.tsx @@ -1,5 +1,5 @@ import { useCallback, useMemo, useState, useEffect } from "react"; -import { View, Pressable, Text, Platform, Alert, ActivityIndicator } from "react-native"; +import { View, Pressable, Text, Platform } from "react-native"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import Animated, { useAnimatedStyle, @@ -9,21 +9,17 @@ import Animated, { } from "react-native-reanimated"; import { Gesture, GestureDetector } from "react-native-gesture-handler"; import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles"; -import { Plus, Settings, Users, AudioLines } from "lucide-react-native"; +import { Plus, Settings, Users } from "lucide-react-native"; import { router } from "expo-router"; import { usePanelStore } from "@/stores/panel-store"; import { GroupedAgentList } from "./grouped-agent-list"; import { useAggregatedAgents } from "@/hooks/use-aggregated-agents"; import { useSidebarAnimation } from "@/contexts/sidebar-animation-context"; import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window"; -import { useVoice } from "@/contexts/voice-context"; -import { useDaemonConnections } from "@/contexts/daemon-connections-context"; -import { VoicePanel } from "./voice-panel"; import { useSidebarAgentSections } from "@/hooks/use-sidebar-agent-sections"; import { useSidebarCollapsedSectionsStore } from "@/stores/sidebar-collapsed-sections-store"; import { useKeyboardNavStore } from "@/stores/keyboard-nav-store"; import { deriveSidebarShortcutAgentKeys } from "@/utils/sidebar-shortcuts"; -import { AdaptiveModalSheet } from "@/components/adaptive-modal-sheet"; const DESKTOP_SIDEBAR_WIDTH = 320; const SIDEBAR_AGENT_LIMIT = 50; // temporary for screenshots @@ -56,9 +52,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { } = useSidebarAnimation(); const trafficLightPadding = useTrafficLightPadding(); const dragHandlers = useTauriDragHandlers(); - const { connectionStates } = useDaemonConnections(); - const { isVoiceMode, isVoiceSwitching, startVoice, stopVoice } = useVoice(); - const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false); // Track user-initiated refresh to avoid showing spinner on background revalidation const [isManualRefresh, setIsManualRefresh] = useState(false); @@ -142,65 +135,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { router.push("/agents"); }, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]); - const voiceEligibleHosts = useMemo(() => { - return Array.from(connectionStates.values()).filter((entry) => entry.status === "online"); - }, [connectionStates]); - const hasAnyConfiguredHosts = connectionStates.size > 0; - - const handleToggleVoice = useCallback(() => { - if (isVoiceSwitching) { - return; - } - if (isVoiceMode) { - void stopVoice().catch((error) => { - console.error("[SlidingSidebar] Failed to stop voice", error); - Alert.alert("Voice failed", "Unable to stop voice mode."); - }); - return; - } - - if (voiceEligibleHosts.length === 0) { - if (!hasAnyConfiguredHosts) { - Alert.alert( - "No hosts available", - "Add a host in Settings before starting Voice mode.", - [{ text: "Open Settings", onPress: () => router.push("/settings") }, { text: "OK" }] - ); - return; - } - Alert.alert( - "Hosts reconnecting", - "Every host is offline right now. Paseo reconnects automatically—try Voice mode again once one comes online." - ); - return; - } - - if (voiceEligibleHosts.length === 1) { - void startVoice(voiceEligibleHosts[0].daemon.serverId).catch((error) => { - console.error("[SlidingSidebar] Failed to start voice", error); - Alert.alert("Voice failed", "Unable to start Voice mode for this host."); - }); - return; - } - - setShowVoiceHostPicker(true); - }, [hasAnyConfiguredHosts, isVoiceMode, isVoiceSwitching, startVoice, stopVoice, voiceEligibleHosts]); - - const handleSelectVoiceHost = useCallback( - (serverId: string) => { - setShowVoiceHostPicker(false); - void startVoice(serverId).catch((error) => { - console.error("[SlidingSidebar] Failed to start voice", error); - Alert.alert("Voice failed", "Unable to start Voice mode for this host."); - }); - }, - [startVoice] - ); - - const handleDismissVoiceHostPicker = useCallback(() => { - setShowVoiceHostPicker(false); - }, []); - // Close gesture (swipe left to close when sidebar is open) // Only activates on leftward swipe, fails on rightward or vertical movement // This mirrors the explorer-sidebar pattern for the right sidebar @@ -301,8 +235,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { parentGestureRef={closeGestureRef} /> - {isVoiceMode ? : null} - {/* Footer */} - - {({ hovered }) => ( - isVoiceSwitching ? ( - - ) : ( - - ) - )} - - - - - {voiceEligibleHosts.map((entry) => ( - handleSelectVoiceHost(entry.daemon.serverId)} - > - {entry.daemon.label} - - ))} - - Cancel - - - ); } @@ -439,8 +308,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { selectedAgentId={selectedAgentId} /> - {isVoiceMode ? : null} - {/* Footer */} - - {({ hovered }) => ( - isVoiceSwitching ? ( - - ) : ( - - ) - )} - - - - - {voiceEligibleHosts.map((entry) => ( - handleSelectVoiceHost(entry.daemon.serverId)} - > - {entry.daemon.label} - - ))} - - Cancel - - - ); } diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index 0b66c243f..fd2fd2692 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -18,10 +18,12 @@ interface VoiceContextValue { isDetecting: boolean; isSpeaking: boolean; segmentDuration: number; - startVoice: (serverId: string) => Promise; + startVoice: (serverId: string, agentId: string) => Promise; stopVoice: () => Promise; + isVoiceModeForAgent: (serverId: string, agentId: string) => boolean; toggleMute: () => void; activeServerId: string | null; + activeAgentId: string | null; } const VoiceContext = createContext(null); @@ -45,6 +47,7 @@ interface VoiceProviderProps { export function VoiceProvider({ children }: VoiceProviderProps) { const getSession = useSessionStore((state) => state.getSession); const [activeServerId, setActiveServerId] = useState(null); + const [activeAgentId, setActiveAgentId] = useState(null); const activeSession = useSessionStore( useCallback( (state: ReturnType) => { @@ -151,7 +154,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) { voiceTransportReadyRef.current = false; } - if (!isVoiceMode || !activeServerId || !client) { + if (!isVoiceMode || !activeServerId || !activeAgentId || !client) { wasVoiceSocketConnectedRef.current = connected; if (!isVoiceMode) { lastVoiceModeSyncedClientRef.current = null; @@ -167,7 +170,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) { voiceResyncInFlightRef.current = true; voiceTransportReadyRef.current = false; setIsVoiceSwitching(true); - void client.setVoiceMode(true).then( + void client.setVoiceMode(true, activeAgentId).then( () => { console.log("[Voice] Re-synced voice mode after reconnect"); lastVoiceModeSyncedClientRef.current = client; @@ -184,7 +187,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) { } wasVoiceSocketConnectedRef.current = connected; - }, [activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]); + }, [activeAgentId, activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]); const isPlayingAudio = activeSession?.isPlayingAudio ?? false; @@ -201,7 +204,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) { }, [isPlayingAudio]); const startVoice = useCallback( - async (serverId: string) => { + async (serverId: string, agentId: string) => { const session = getSession(serverId) ?? null; if (!session) { throw new Error(`Host ${serverId} is not connected`); @@ -210,13 +213,23 @@ export function VoiceProvider({ children }: VoiceProviderProps) { setIsVoiceSwitching(true); voiceTransportReadyRef.current = false; try { + const previousSession = realtimeSessionRef.current; + if ( + isVoiceMode && + previousSession?.client && + (activeServerId !== serverId || activeAgentId !== agentId) + ) { + await previousSession.client.setVoiceMode(false); + } + realtimeSessionRef.current = session; setActiveServerId(serverId); + setActiveAgentId(agentId); await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => { console.warn("[Voice] Failed to activate keep-awake:", error); }); if (session?.client) { - await session.client.setVoiceMode(true); + await session.client.setVoiceMode(true, agentId); } else { console.warn("[Voice] setVoiceMode skipped: daemon unavailable"); } @@ -230,13 +243,14 @@ export function VoiceProvider({ children }: VoiceProviderProps) { console.error("[Voice] Failed to start:", error); await realtimeAudio.stop().catch(() => undefined); setActiveServerId((current) => (current === serverId ? null : current)); + setActiveAgentId((current) => (current === agentId ? null : current)); await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined); throw error; } finally { setIsVoiceSwitching(false); } }, - [getSession, realtimeAudio] + [activeAgentId, activeServerId, getSession, isVoiceMode, realtimeAudio] ); const stopVoice = useCallback(async () => { @@ -254,6 +268,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) { await realtimeAudio.stop(); setIsVoiceMode(false); setActiveServerId(null); + setActiveAgentId(null); await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined); console.log("[Voice] Mode disabled"); } catch (error: any) { @@ -265,6 +280,12 @@ export function VoiceProvider({ children }: VoiceProviderProps) { } }, [realtimeAudio]); + const isVoiceModeForAgent = useCallback( + (serverId: string, agentId: string) => + isVoiceMode && activeServerId === serverId && activeAgentId === agentId, + [activeAgentId, activeServerId, isVoiceMode] + ); + const value: VoiceContextValue = { isVoiceMode, isVoiceSwitching, @@ -275,8 +296,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) { segmentDuration: realtimeAudio.segmentDuration, startVoice, stopVoice, + isVoiceModeForAgent, toggleMute: realtimeAudio.toggleMute, activeServerId, + activeAgentId, }; return ( diff --git a/packages/server/scripts/voice-roundtrip-debug.ts b/packages/server/scripts/voice-roundtrip-debug.ts index 467ff1097..4c1beef8d 100644 --- a/packages/server/scripts/voice-roundtrip-debug.ts +++ b/packages/server/scripts/voice-roundtrip-debug.ts @@ -1,4 +1,6 @@ -import { randomUUID } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import pino from "pino"; import { createTestPaseoDaemon } from "../src/server/test-utils/paseo-daemon.js"; @@ -48,9 +50,20 @@ async function main(): Promise { await client.connect(); client.subscribeAgentUpdates({ subscriptionId: "voice-debug" }); - const voiceAgentId = randomUUID(); + const voiceCwd = mkdtempSync(path.join(tmpdir(), "voice-roundtrip-debug-")); + const voiceAgent = await client.createAgent({ + config: { + provider: "claude", + cwd: voiceCwd, + modeId: "bypassPermissions", + }, + }); + const voiceAgentId = voiceAgent.id; const mode = await client.setVoiceMode(true, voiceAgentId); console.log("set_voice_mode_response", mode); + if (!mode.accepted) { + throw new Error(`setVoiceMode rejected: ${mode.error ?? "unknown error"}`); + } const offTranscript = client.on("transcription_result", (msg) => { if (msg.type !== "transcription_result") return; @@ -149,6 +162,7 @@ async function main(): Promise { console.log("success", { audioChunkCount }); await client.setVoiceMode(false); + rmSync(voiceCwd, { recursive: true, force: true }); offTranscript(); offActivity(); diff --git a/packages/server/src/client/daemon-client.ts b/packages/server/src/client/daemon-client.ts index a3ebb15d3..b94df1d38 100644 --- a/packages/server/src/client/daemon-client.ts +++ b/packages/server/src/client/daemon-client.ts @@ -1291,13 +1291,13 @@ export class DaemonClient { async setVoiceMode( enabled: boolean, - voiceAgentId?: string + agentId?: string ): Promise { const requestId = this.createRequestId(); const message = SessionInboundMessageSchema.parse({ type: "set_voice_mode", enabled, - ...(voiceAgentId ? { voiceAgentId } : {}), + ...(agentId ? { agentId } : {}), requestId, }); return this.sendRequest({ diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index efb939c15..0c5fe67f7 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -407,22 +407,24 @@ export class AgentManager { ); } - async refreshAgentFromPersistence(agentId: string): Promise { + async refreshAgentFromPersistence( + agentId: string, + overrides?: Partial + ): Promise { const existing = this.requireAgent(agentId); const handle = existing.persistence; - if (!handle) { - throw new Error( - `Agent ${agentId} cannot be refreshed because it has no persistence handle` - ); - } - - const client = this.requireClient(handle.provider); - const overrides = { + const provider = handle?.provider ?? existing.provider; + const client = this.requireClient(provider); + const refreshConfig = { ...existing.config, - provider: handle.provider, - }; + ...overrides, + provider, + } as AgentSessionConfig; + const normalizedConfig = await this.normalizeConfig(refreshConfig); - const session = await client.resumeSession(handle, overrides); + const session = handle + ? await client.resumeSession(handle, normalizedConfig) + : await client.createSession(normalizedConfig); // Remove the existing agent entry before swapping sessions this.agents.delete(agentId); @@ -436,7 +438,12 @@ export class AgentManager { } // Preserve existing labels during refresh - return this.registerSession(session, overrides, agentId, { labels: existing.labels }); + return this.registerSession(session, normalizedConfig, agentId, { + labels: existing.labels, + createdAt: existing.createdAt, + updatedAt: existing.updatedAt, + lastUserMessageAt: existing.lastUserMessageAt, + }); } async closeAgent(agentId: string): Promise { diff --git a/packages/server/src/server/agent/mcp-server.ts b/packages/server/src/server/agent/mcp-server.ts index b8c62667e..24cc0ef38 100644 --- a/packages/server/src/server/agent/mcp-server.ts +++ b/packages/server/src/server/agent/mcp-server.ts @@ -57,6 +57,7 @@ export interface AgentMcpServerOptions { callerAgentId: string ) => VoiceCallerContext | null; enableVoiceTools?: boolean; + voiceOnly?: boolean; logger: Logger; } @@ -393,7 +394,7 @@ export async function createAgentMcpServer( initialMode: topLevelInputSchema.initialMode.optional(), }); - if (options.enableVoiceTools || callerContext?.enableVoiceTools) { + if (options.voiceOnly || options.enableVoiceTools || callerContext?.enableVoiceTools) { server.registerTool( "speak", { @@ -432,6 +433,10 @@ export async function createAgentMcpServer( ); } + if (options.voiceOnly) { + return server; + } + server.registerTool( "create_agent", { diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index d92db4a2b..37f57a287 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -64,7 +64,6 @@ import type { AgentClient, AgentProvider, } from "./agent/agent-sdk-types.js"; -import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js"; import { acquirePidLock, releasePidLock } from "./pid-lock.js"; import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js"; import { @@ -278,98 +277,8 @@ export async function createPaseoDaemon( logger.info( `Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand` ); - - const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null; - const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false; - const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS - .filter((definition) => definition.voice?.enabled) - .map((definition) => definition.id as AgentProvider); logger.info( - { - requestedVoiceLlmProvider, - voiceLlmProviderExplicit, - voiceEnabledProviders, - }, - "Voice LLM provider reconciliation started" - ); - - const providerClients = createAllClients(logger); - Object.assign(providerClients, config.agentClients); - const voiceLlmAvailability = Object.fromEntries( - voiceEnabledProviders.map((provider) => [provider, false]) - ) as Record; - for (const provider of voiceEnabledProviders) { - try { - voiceLlmAvailability[provider] = await providerClients[provider].isAvailable(); - } catch (error) { - logger.warn({ err: error, provider }, "Voice LLM provider availability check failed"); - voiceLlmAvailability[provider] = false; - } - } - - let resolvedVoiceLlmProvider: AgentProvider | null = null; - if (requestedVoiceLlmProvider) { - if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) { - logger.error( - { provider: requestedVoiceLlmProvider, voiceEnabledProviders }, - "Configured voice LLM provider does not support voice mode" - ); - throw new Error( - `Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode` - ); - } - if (!voiceLlmAvailability[requestedVoiceLlmProvider]) { - logger.error( - { provider: requestedVoiceLlmProvider, voiceLlmAvailability }, - "Configured voice LLM provider is unavailable" - ); - throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`); - } - resolvedVoiceLlmProvider = requestedVoiceLlmProvider; - } else { - resolvedVoiceLlmProvider = - voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null; - } - - let resolvedVoiceLlmModeId: string | null = null; - let resolvedVoiceLlmModel: string | null = null; - - if (!resolvedVoiceLlmProvider) { - if (voiceLlmProviderExplicit) { - logger.error( - { requestedVoiceLlmProvider, voiceLlmAvailability }, - "No voice LLM provider available" - ); - throw new Error("No voice LLM provider available"); - } - logger.warn( - { requestedVoiceLlmProvider, voiceLlmAvailability }, - "No default voice LLM provider available; voice mode will be disabled until a provider is configured" - ); - } else { - const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find( - (definition) => definition.id === resolvedVoiceLlmProvider - ); - if (!resolvedVoiceProviderDefinition?.voice?.enabled) { - throw new Error( - `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry` - ); - } - resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId; - resolvedVoiceLlmModel = - config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null; - } - - logger.info( - { - requestedVoiceLlmProvider, - voiceLlmProviderExplicit, - resolvedVoiceLlmProvider, - resolvedVoiceLlmModeId, - resolvedVoiceLlmModel, - voiceLlmAvailability, - }, - "Voice LLM provider reconciliation completed" + "Voice mode configured for agent-scoped resume flow (no dedicated voice assistant provider)" ); let wsServer: VoiceAssistantWebSocketServer | null = null; let voiceMcpBridgeManager: VoiceMcpSocketBridgeManager | null = null; @@ -523,7 +432,7 @@ export async function createPaseoDaemon( agentStorage, paseoHome: config.paseoHome, callerAgentId, - enableVoiceTools: false, + voiceOnly: true, resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null, resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null, logger, @@ -555,10 +464,6 @@ export async function createPaseoDaemon( { stt: sttService, tts: ttsService }, terminalManager, { - voiceLlmProvider: resolvedVoiceLlmProvider, - voiceLlmModeId: resolvedVoiceLlmModeId, - voiceLlmProviderExplicit, - voiceLlmModel: resolvedVoiceLlmModel, voiceAgentMcpStdio: { command: voiceMcpBridgeCommand.command, baseArgs: [...voiceMcpBridgeCommand.baseArgs], diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index a2c371db6..449086e48 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -142,23 +142,29 @@ describe("daemon client E2E", () => { const agents = await ctx.client.fetchAgents(); expect(Array.isArray(agents)).toBe(true); - const voiceAgents = await ctx.client.fetchAgents({ - filter: { labels: { surface: "voice" } }, + const cwd = tmpCwd(); + const created = await ctx.client.createAgent({ + config: { + ...getFullAccessConfig("codex"), + cwd, + }, }); - expect(Array.isArray(voiceAgents)).toBe(true); - await expect(ctx.client.setVoiceMode(true)).resolves.toMatchObject({ + await expect(ctx.client.setVoiceMode(true, created.id)).resolves.toMatchObject({ enabled: true, + agentId: created.id, accepted: true, error: null, }); await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({ enabled: false, + agentId: null, accepted: true, error: null, }); await ctx.client.deleteAgent(randomUUID()); + rmSync(cwd, { recursive: true, force: true }); }, 30000); test("emits server_info on websocket connect", async () => { @@ -655,7 +661,14 @@ describe("daemon client E2E", () => { speechTest( "voice mode buffers audio until isLast and emits transcription_result", async () => { - await ctx.client.setVoiceMode(true, randomUUID()); + const voiceCwd = tmpCwd(); + const voiceAgent = await ctx.client.createAgent({ + config: { + ...getFullAccessConfig("codex"), + cwd: voiceCwd, + }, + }); + await ctx.client.setVoiceMode(true, voiceAgent.id); const transcription = waitForSignal(30_000, (resolve) => { const unsubscribe = ctx.client.on("transcription_result", (message) => { @@ -751,6 +764,7 @@ describe("daemon client E2E", () => { } finally { await Promise.allSettled([transcription, errorSignal]); await ctx.client.setVoiceMode(false); + rmSync(voiceCwd, { recursive: true, force: true }); } }, 90_000 diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 672534059..5ec1379a1 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1,5 +1,5 @@ import { v4 as uuidv4 } from "uuid"; -import { mkdir, stat } from "fs/promises"; +import { stat } from "fs/promises"; import { exec } from "child_process"; import { promisify } from "util"; import { join, resolve, sep } from "path"; @@ -58,6 +58,7 @@ import type { AgentPermissionResponse, AgentPromptContentBlock, AgentPromptInput, + McpServerConfig, AgentSessionConfig, AgentStreamEvent, AgentProvider, @@ -191,7 +192,18 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round( PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS ); const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/; -const VoiceAgentIdSchema = z.string().uuid(); +const AgentIdSchema = z.string().uuid(); +const VOICE_MCP_SERVER_NAME = "paseo_voice"; +const VOICE_MODE_ENFORCED_MODE_ID: Partial> = { + claude: "default", + codex: "read-only", +}; + +type VoiceModeBaseConfig = { + systemPrompt?: string; + mcpServers?: Record; + modeId?: string; +}; interface AudioBufferState { chunks: Buffer[]; @@ -214,10 +226,6 @@ export type SessionOptions = { tts: TextToSpeechProvider | null; terminalManager: TerminalManager | null; voice?: { - voiceLlmProvider?: AgentProvider | null; - voiceLlmModeId?: string | null; - voiceLlmProviderExplicit?: boolean; - voiceLlmModel?: string | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; }; voiceBridge?: { @@ -376,10 +384,6 @@ export class Session { } | null = null; private readonly terminalManager: TerminalManager | null; private terminalSubscriptions: Map void> = new Map(); - private readonly voiceLlmProvider: AgentProvider | null; - private readonly voiceLlmModeId: string | null; - private readonly voiceLlmProviderExplicit: boolean; - private readonly voiceLlmModel: string | null; private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null; private readonly localSpeechModelsDir: string; private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[]; @@ -395,7 +399,8 @@ export class Session { private readonly unregisterVoiceCallerContext?: (agentId: string) => void; private readonly ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise; private readonly removeVoiceMcpSocketForAgent?: (agentId: string) => Promise; - private voiceAssistantAgentId: string | null = null; + private voiceModeAgentId: string | null = null; + private voiceModeBaseConfig: VoiceModeBaseConfig | null = null; constructor(options: SessionOptions) { const { @@ -425,10 +430,6 @@ export class Session { this.agentStorage = agentStorage; this.createAgentMcpTransport = createAgentMcpTransport; this.terminalManager = terminalManager; - this.voiceLlmProvider = voice?.voiceLlmProvider ?? null; - this.voiceLlmModeId = voice?.voiceLlmModeId ?? null; - this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false; - this.voiceLlmModel = voice?.voiceLlmModel ?? null; this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null; const configuredModelsDir = dictation?.localModels?.modelsDir?.trim(); this.localSpeechModelsDir = @@ -896,7 +897,7 @@ export class Session { break; case "set_voice_mode": - await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId, msg.requestId); + await this.handleSetVoiceMode(msg.enabled, msg.agentId, msg.requestId); break; case "send_agent_message_request": @@ -1276,108 +1277,216 @@ export class Session { */ private async handleSetVoiceMode( enabled: boolean, - voiceAgentId?: string, + agentId?: string, requestId?: string ): Promise { - if (enabled) { - let normalizedVoiceAgentId: string | null = null; - if (voiceAgentId && voiceAgentId.trim().length > 0) { - normalizedVoiceAgentId = this.parseVoiceAgentId( - voiceAgentId, + try { + if (enabled) { + const normalizedAgentId = this.parseVoiceTargetAgentId( + agentId ?? "", "set_voice_mode" ); - } else { - normalizedVoiceAgentId = await this.findExistingVoiceAssistantAgentId(); + + if ( + this.isVoiceMode && + this.voiceModeAgentId && + this.voiceModeAgentId !== normalizedAgentId + ) { + await this.disableVoiceModeForActiveAgent(true); + } + + if (!this.isVoiceMode || this.voiceModeAgentId !== normalizedAgentId) { + const refreshedAgentId = await this.enableVoiceModeForAgent(normalizedAgentId); + this.voiceModeAgentId = refreshedAgentId; + } + + this.isVoiceMode = true; + this.sessionLogger.info( + { + agentId: this.voiceModeAgentId, + }, + "Voice mode enabled for existing agent" + ); + if (requestId) { + this.emit({ + type: "set_voice_mode_response", + payload: { + requestId, + enabled: true, + agentId: this.voiceModeAgentId, + accepted: true, + error: null, + }, + }); + } + return; } - this.isVoiceMode = true; - this.voiceAssistantAgentId = normalizedVoiceAgentId; - this.sessionLogger.info( + await this.disableVoiceModeForActiveAgent(true); + this.isVoiceMode = false; + this.sessionLogger.info("Voice mode disabled"); + if (requestId) { + this.emit({ + type: "set_voice_mode_response", + payload: { + requestId, + enabled: false, + agentId: null, + accepted: true, + error: null, + }, + }); + } + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Failed to set voice mode"; + this.sessionLogger.error( { - voiceAssistantAgentId: this.voiceAssistantAgentId, - resumedVoiceAgent: Boolean(normalizedVoiceAgentId), - providedVoiceAgentId: Boolean(voiceAgentId?.trim()), + err: error, + enabled, + requestedAgentId: agentId ?? null, }, - "Voice mode enabled (agent-backed)" + "set_voice_mode failed" ); if (requestId) { this.emit({ type: "set_voice_mode_response", payload: { requestId, - enabled: true, - voiceAgentId: this.voiceAssistantAgentId, - accepted: true, - error: null, + enabled: this.isVoiceMode, + agentId: this.voiceModeAgentId, + accepted: false, + error: errorMessage, }, }); + return; } - return; - } - - this.isVoiceMode = false; - const disabledVoiceAgentId = this.voiceAssistantAgentId; - if (disabledVoiceAgentId && this.removeVoiceMcpSocketForAgent) { - try { - await this.removeVoiceMcpSocketForAgent(disabledVoiceAgentId); - } catch (error) { - this.sessionLogger.warn( - { err: error, voiceAssistantAgentId: disabledVoiceAgentId }, - "Failed to remove voice MCP socket bridge on disable" - ); - } - } - this.sessionLogger.info( - { voiceAssistantAgentId: this.voiceAssistantAgentId }, - "Voice mode disabled (agent-backed)" - ); - if (requestId) { - this.emit({ - type: "set_voice_mode_response", - payload: { - requestId, - enabled: false, - voiceAgentId: this.voiceAssistantAgentId, - accepted: true, - error: null, - }, - }); + throw error; } } - private parseVoiceAgentId(rawId: string, source: string): string { - const parsed = VoiceAgentIdSchema.safeParse(rawId.trim()); + private parseVoiceTargetAgentId(rawId: string, source: string): string { + const parsed = AgentIdSchema.safeParse(rawId.trim()); if (!parsed.success) { - throw new Error(`${source}: voiceAgentId must be a UUID`); + throw new Error(`${source}: agentId must be a UUID`); } return parsed.data; } - private async findExistingVoiceAssistantAgentId(): Promise { - const scopedCandidates = await this.listAgentPayloads({ - labels: { - surface: "voice", - voiceClientId: this.clientId, - }, - }); - const fallbackCandidates = await this.listAgentPayloads({ - labels: { surface: "voice" }, - }); - const available = [ - ...scopedCandidates, - ...fallbackCandidates.filter( - (candidate) => !scopedCandidates.some((scoped) => scoped.id === candidate.id) - ), - ].filter((agent) => !agent.archivedAt && agent.labels?.ui !== "false"); - if (available.length === 0) { - return null; + private cloneMcpServers( + servers: Record | undefined + ): Record | undefined { + if (!servers) { + return undefined; } - available.sort((left, right) => { - const leftTime = Date.parse(left.updatedAt); - const rightTime = Date.parse(right.updatedAt); - return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0); + return JSON.parse(JSON.stringify(servers)) as Record; + } + + private buildVoiceModeSystemPrompt(existing?: string): string { + const chunks = [existing?.trim(), VOICE_AGENT_SYSTEM_INSTRUCTION].filter( + (value): value is string => Boolean(value && value.length > 0) + ); + return chunks.join("\n\n"); + } + + private buildVoiceModeMcpServers( + existing: Record | undefined, + socketPath: string + ): Record { + const mcpStdio = this.voiceAgentMcpStdio; + if (!mcpStdio) { + throw new Error("Voice MCP stdio bridge is not configured"); + } + return { + ...(existing ?? {}), + [VOICE_MCP_SERVER_NAME]: buildVoiceAgentMcpServerConfig({ + command: mcpStdio.command, + baseArgs: mcpStdio.baseArgs, + socketPath, + env: mcpStdio.env, + }), + }; + } + + private getVoiceModeEnforcedModeId(provider: AgentProvider): string | undefined { + return VOICE_MODE_ENFORCED_MODE_ID[provider]; + } + + private async enableVoiceModeForAgent(agentId: string): Promise { + const ensureVoiceSocket = this.ensureVoiceMcpSocketForAgent; + if (!ensureVoiceSocket) { + throw new Error("Voice MCP socket bridge is not configured"); + } + + const existing = await this.ensureAgentLoaded(agentId); + + const socketPath = await ensureVoiceSocket(agentId); + this.registerVoiceBridgeForAgent(agentId); + + const baseConfig: VoiceModeBaseConfig = { + systemPrompt: existing.config.systemPrompt, + mcpServers: this.cloneMcpServers(existing.config.mcpServers), + modeId: existing.config.modeId, + }; + this.voiceModeBaseConfig = baseConfig; + const voiceModeId = this.getVoiceModeEnforcedModeId(existing.config.provider); + const refreshOverrides: Partial = { + systemPrompt: this.buildVoiceModeSystemPrompt(baseConfig.systemPrompt), + mcpServers: this.buildVoiceModeMcpServers(baseConfig.mcpServers, socketPath), + ...(voiceModeId ? { modeId: voiceModeId } : {}), + }; + + try { + const refreshed = await this.agentManager.refreshAgentFromPersistence( + agentId, + refreshOverrides + ); + return refreshed.id; + } catch (error) { + this.unregisterVoiceSpeakHandler?.(agentId); + this.unregisterVoiceCallerContext?.(agentId); + await this.removeVoiceMcpSocketForAgent?.(agentId).catch(() => undefined); + this.voiceModeBaseConfig = null; + throw error; + } + } + + private async disableVoiceModeForActiveAgent( + restoreAgentConfig: boolean + ): Promise { + const agentId = this.voiceModeAgentId; + if (!agentId) { + this.voiceModeBaseConfig = null; + return; + } + + this.unregisterVoiceSpeakHandler?.(agentId); + this.unregisterVoiceCallerContext?.(agentId); + await this.removeVoiceMcpSocketForAgent?.(agentId).catch((error) => { + this.sessionLogger.warn( + { err: error, agentId }, + "Failed to remove voice MCP socket bridge on disable" + ); }); - return available[0]?.id ?? null; + + if (restoreAgentConfig && this.voiceModeBaseConfig) { + const baseConfig = this.voiceModeBaseConfig; + try { + await this.agentManager.refreshAgentFromPersistence(agentId, { + systemPrompt: baseConfig.systemPrompt, + mcpServers: this.cloneMcpServers(baseConfig.mcpServers), + modeId: baseConfig.modeId, + }); + } catch (error) { + this.sessionLogger.warn( + { err: error, agentId }, + "Failed to restore agent config while disabling voice mode" + ); + } + } + + this.voiceModeBaseConfig = null; + this.voiceModeAgentId = null; } /** @@ -4501,101 +4610,11 @@ export class Session { } } - private async ensureVoiceAssistantAgent(): Promise { - const ensureVoiceSocket = this.ensureVoiceMcpSocketForAgent; - if (!ensureVoiceSocket) { - throw new Error("Voice MCP socket bridge is not configured"); - } - - if (this.voiceAssistantAgentId) { - const existing = this.agentManager.getAgent(this.voiceAssistantAgentId); - if (existing) { - await ensureVoiceSocket(existing.id); - this.registerVoiceBridgeForAgent(existing.id); - return existing.id; - } - try { - const hydrated = await this.ensureAgentLoaded(this.voiceAssistantAgentId); - await ensureVoiceSocket(hydrated.id); - this.voiceAssistantAgentId = hydrated.id; - this.registerVoiceBridgeForAgent(hydrated.id); - return hydrated.id; - } catch (error) { - this.sessionLogger.debug( - { err: error, voiceAssistantAgentId: this.voiceAssistantAgentId }, - "Voice assistant agent not found in active/persisted state; creating new session" - ); - } - } - - const provider = this.voiceLlmProvider; - if (!provider) { - throw new Error("Voice LLM provider is not configured"); - } - const voiceAgentId = this.voiceAssistantAgentId ?? uuidv4(); - const cwd = join(this.paseoHome, "voice-agent-workspace"); - await mkdir(cwd, { recursive: true }); - - const mcpStdio = this.voiceAgentMcpStdio; - if (!mcpStdio) { - throw new Error("Voice MCP stdio bridge is not configured"); - } - const socketPath = await ensureVoiceSocket(voiceAgentId); - - const model = this.voiceLlmModel?.trim() || undefined; - const config: AgentSessionConfig = { - provider, - cwd, - systemPrompt: VOICE_AGENT_SYSTEM_INSTRUCTION, - modeId: this.voiceLlmModeId ?? "default", - ...(model ? { model } : {}), - mcpServers: { - paseo: buildVoiceAgentMcpServerConfig({ - command: mcpStdio.command, - baseArgs: mcpStdio.baseArgs, - socketPath, - env: mcpStdio.env, - }), - }, - }; - - const created = await this.agentManager.createAgent(config, voiceAgentId, { - labels: { - surface: "voice", - ui: "true", - voiceClientId: this.clientId, - }, - }); - this.voiceAssistantAgentId = created.id; - - try { - this.registerVoiceBridgeForAgent(created.id); - } catch (error) { - this.unregisterVoiceSpeakHandler?.(created.id); - this.unregisterVoiceCallerContext?.(created.id); - await this.removeVoiceMcpSocketForAgent?.(created.id).catch(() => undefined); - await this.agentManager.closeAgent(created.id).catch(() => undefined); - this.voiceAssistantAgentId = null; - throw error; - } - - this.sessionLogger.info( - { - voiceAssistantAgentId: created.id, - provider, - model: model ?? null, - providerExplicit: this.voiceLlmProviderExplicit, - }, - "Voice assistant agent initialized" - ); - return created.id; - } - private registerVoiceBridgeForAgent(agentId: string): void { this.registerVoiceSpeakHandler?.(agentId, async ({ text, signal }) => { this.sessionLogger.info( { - voiceAssistantAgentId: agentId, + agentId, textLength: text.length, preview: text.slice(0, 160), }, @@ -4609,7 +4628,7 @@ export class Session { true ); this.sessionLogger.info( - { voiceAssistantAgentId: agentId, textLength: text.length }, + { agentId, textLength: text.length }, "Voice speak tool call finished playback" ); this.emit({ @@ -4625,7 +4644,7 @@ export class Session { this.registerVoiceCallerContext?.(agentId, { childAgentDefaultLabels: { ui: "true" }, - allowCustomCwd: true, + allowCustomCwd: false, enableVoiceTools: true, }); } @@ -4643,14 +4662,16 @@ export class Session { return isVoicePermissionAllowed(request); } - private async processWithVoiceAgent(userText: string): Promise { - const agentId = await this.ensureVoiceAssistantAgent(); - + private async processWithVoiceAgent( + agentId: string, + userText: string + ): Promise { + await this.ensureAgentLoaded(agentId); await this.interruptAgentIfRunning(agentId); this.sessionLogger.info( { - voiceAssistantAgentId: agentId, + agentId, userTextLength: userText.length, userTextPreview: userText.slice(0, 160), }, @@ -4670,7 +4691,7 @@ export class Session { if (event.item.type === "tool_call" && typeof event.item.name === "string") { this.sessionLogger.info( { - voiceAssistantAgentId: agentId, + agentId, toolName: event.item.name, callId: event.item.callId, }, @@ -4689,11 +4710,12 @@ export class Session { } else { await this.agentManager.respondToPermission(agentId, event.request.id, { behavior: "deny", - message: "Voice assistant policy only allows MCP paseo tools.", + message: "Voice assistant policy only allows the speak tool.", interrupt: true, }); - throw new Error( - `Voice assistant denied non-MCP tool request: ${event.request.name}` + this.sessionLogger.warn( + { agentId, requestName: event.request.name, requestId: event.request.id }, + "Voice assistant denied non-speak permission request" ); } } @@ -4701,12 +4723,12 @@ export class Session { if (!sawSpeakToolCall) { this.sessionLogger.warn( - { voiceAssistantAgentId: agentId }, + { agentId }, "Voice agent turn completed without speak tool call; no audio playback emitted" ); } else { this.sessionLogger.info( - { voiceAssistantAgentId: agentId }, + { agentId }, "Voice agent turn included speak tool call" ); } @@ -4721,11 +4743,18 @@ export class Session { this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled"); return; } + const agentId = this.voiceModeAgentId; + if (!agentId) { + this.sessionLogger.warn( + "Ignoring processVoiceTurn call because no agent is currently voice-enabled" + ); + return; + } const normalized = (latestUserText ?? "").trim(); if (!normalized) { return; } - await this.processWithVoiceAgent(normalized); + await this.processWithVoiceAgent(agentId, normalized); } finally { this.turnIndex++; this.currentStreamPromise = null; @@ -4980,21 +5009,8 @@ export class Session { this.agentTools = null; } - if (this.voiceAssistantAgentId) { - try { - await this.agentManager.closeAgent(this.voiceAssistantAgentId); - } catch (error) { - this.sessionLogger.warn( - { err: error, voiceAssistantAgentId: this.voiceAssistantAgentId }, - "Failed to close voice assistant agent" - ); - } finally { - this.unregisterVoiceSpeakHandler?.(this.voiceAssistantAgentId); - this.unregisterVoiceCallerContext?.(this.voiceAssistantAgentId); - await this.removeVoiceMcpSocketForAgent?.(this.voiceAssistantAgentId).catch(() => undefined); - this.voiceAssistantAgentId = null; - } - } + await this.disableVoiceModeForActiveAgent(true); + this.isVoiceMode = false; // Unsubscribe from all terminals for (const unsubscribe of this.terminalSubscriptions.values()) { diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index 8af66a672..f567ad39b 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -1,8 +1,7 @@ import { describe, expect, test } from "vitest"; -import { mkdtempSync } from "node:fs"; +import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; -import { randomUUID } from "node:crypto"; import pino from "pino"; import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; @@ -191,7 +190,17 @@ describe("speech models (download E2E)", () => { }; }); - await ctx.client.setVoiceMode(true, randomUUID()); + const voiceCwd = mkdtempSync(path.join(tmpdir(), "speech-download-voice-agent-")); + const voiceAgent = await ctx.client.createAgent({ + config: { + provider: "codex", + cwd: voiceCwd, + modeId: "full-access", + model: "gpt-5.1-codex-mini", + thinkingOptionId: "low", + }, + }); + await ctx.client.setVoiceMode(true, voiceAgent.id); for (let offset = 0; offset < pcm16.length; offset += chunkBytes) { const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); const isLast = offset + chunkBytes >= pcm16.length; @@ -202,6 +211,7 @@ describe("speech models (download E2E)", () => { expect(voiceText).toContain("voice note"); } await ctx.client.setVoiceMode(false); + rmSync(voiceCwd, { recursive: true, force: true }); // Streaming TTS: generate locally from downloaded model and validate chunking. const ttsText = "This is a voice note."; diff --git a/packages/server/src/server/voice-local-agent.e2e.test.ts b/packages/server/src/server/voice-local-agent.e2e.test.ts index 795c2afea..066d7230a 100644 --- a/packages/server/src/server/voice-local-agent.e2e.test.ts +++ b/packages/server/src/server/voice-local-agent.e2e.test.ts @@ -1,9 +1,12 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import { randomUUID } from "node:crypto"; import path from "node:path"; -import { readFile } from "node:fs/promises"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import pino from "pino"; import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js"; +import { getFullAccessConfig } from "./daemon-e2e/agent-configs.js"; +import { OpenAITTS } from "./speech/providers/openai/tts.js"; const openaiApiKey = process.env.OPENAI_API_KEY ?? null; const shouldRun = @@ -69,61 +72,94 @@ function waitForSignal( test( "routes voice turns through local agent speak tool", async () => { - await ctx.client.setVoiceMode(true, randomUUID()); - - const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => { - const offAudio = ctx.client.on("audio_output", (msg) => { - if (msg.type !== "audio_output") return; - resolve({ chunkId: msg.payload.id }); - }); - const offError = ctx.client.on("activity_log", (msg) => { - if (msg.type !== "activity_log") return; - if (msg.payload.type !== "error") return; - reject(new Error(String(msg.payload.content))); - }); - return () => { - offAudio(); - offError(); - }; - }); - - const assistantLogPromise = waitForSignal(120000, (resolve, reject) => { - const offLog = ctx.client.on("activity_log", (msg) => { - if (msg.type !== "activity_log") return; - if (msg.payload.type !== "assistant") return; - const content = String(msg.payload.content ?? ""); - if (!content.trim()) return; - resolve(content); - }); - const offError = ctx.client.on("activity_log", (msg) => { - if (msg.type !== "activity_log") return; - if (msg.payload.type !== "error") return; - reject(new Error(String(msg.payload.content))); - }); - return () => { - offLog(); - offError(); - }; - }); - - const fixturePath = path.resolve( - process.cwd(), - "..", - "app", - "e2e", - "fixtures", - "recording.wav" + const logger = pino({ level: "silent" }); + const ttsProvider = new OpenAITTS( + { + apiKey: openaiApiKey!, + responseFormat: "pcm", + voice: "alloy", + }, + logger ); - const wav = await readFile(fixturePath); - await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true); - const [{ chunkId }, assistantText] = await Promise.all([ - audioPromise, - assistantLogPromise, + const voiceCwd = mkdtempSync(path.join(tmpdir(), "voice-local-agent-")); + const targetAgent = await ctx.client.createAgent({ + config: { + ...getFullAccessConfig("codex"), + cwd: voiceCwd, + }, + }); + const voiceMode = await ctx.client.setVoiceMode(true, targetAgent.id); + expect(voiceMode.accepted).toBe(true); + + const transcriptionPromise = waitForSignal(120000, (resolve, reject) => { + const offTranscript = ctx.client.on("transcription_result", (msg) => { + if (msg.type !== "transcription_result") return; + const text = String(msg.payload.text ?? "").trim(); + if (!text) return; + resolve(text); + }); + const offError = ctx.client.on("activity_log", (msg) => { + if (msg.type !== "activity_log") return; + if (msg.payload.type !== "error") return; + reject(new Error(String(msg.payload.content))); + }); + return () => { + offTranscript(); + offError(); + }; + }); + + const speakToolPromise = waitForSignal(120000, (resolve, reject) => { + const offStream = ctx.client.on("agent_stream", (msg) => { + if (msg.type !== "agent_stream") return; + if (msg.payload.event.type !== "timeline") return; + const item = msg.payload.event.item; + if (item.type !== "tool_call") return; + const name = String(item.name ?? ""); + if (!name.toLowerCase().includes("speak")) return; + resolve(name); + }); + const offError = ctx.client.on("activity_log", (msg) => { + if (msg.type !== "activity_log") return; + if (msg.payload.type !== "error") return; + reject(new Error(String(msg.payload.content))); + }); + return () => { + offStream(); + offError(); + }; + }); + + const inputSpeech = await ttsProvider.synthesizeSpeech( + "Use the speak tool and say exactly local agent check." + ); + const buffers: Buffer[] = []; + for await (const chunk of inputSpeech.stream as AsyncIterable) { + buffers.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as ArrayBufferLike)); + } + const pcm = Buffer.concat(buffers); + const chunkBytes = 4800; + for (let offset = 0; offset < pcm.length; offset += chunkBytes) { + const chunk = pcm.subarray(offset, Math.min(pcm.length, offset + chunkBytes)); + const isLast = offset + chunkBytes >= pcm.length; + await ctx.client.sendVoiceAudioChunk( + chunk.toString("base64"), + "audio/pcm;rate=24000;bits=16", + isLast + ); + } + + const [transcript, speakToolName] = await Promise.all([ + transcriptionPromise, + speakToolPromise, ]); - expect(chunkId.length).toBeGreaterThan(0); - expect(assistantText.trim().length).toBeGreaterThan(0); + await ctx.client.setVoiceMode(false).catch(() => undefined); + rmSync(voiceCwd, { recursive: true, force: true }); + + expect(transcript.length).toBeGreaterThan(0); + expect(speakToolName.toLowerCase()).toContain("speak"); const agents = await ctx.client.fetchAgents(); expect( diff --git a/packages/server/src/server/voice-permission-policy.test.ts b/packages/server/src/server/voice-permission-policy.test.ts index 6ca551183..5b62d4fd0 100644 --- a/packages/server/src/server/voice-permission-policy.test.ts +++ b/packages/server/src/server/voice-permission-policy.test.ts @@ -21,13 +21,13 @@ describe("isVoicePermissionAllowed", () => { expect(result).toBe(true); }); - test("allows explicit MCP/paseo tool names", () => { + test("denies explicit MCP/paseo tool names", () => { expect( isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" })) - ).toBe(true); + ).toBe(false); expect( isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" })) - ).toBe(true); + ).toBe(false); }); test("denies non-tool permission kinds", () => { @@ -37,12 +37,12 @@ describe("isVoicePermissionAllowed", () => { expect(result).toBe(false); }); - test("allows codextool only when metadata references MCP/paseo", () => { + test("allows codextool only when metadata references speak", () => { const allowed = isVoicePermissionAllowed( buildRequest({ name: "codextool", metadata: { - questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }], + questions: [{ question: "Allow codextool to call speak for user feedback?" }], }, }) ); diff --git a/packages/server/src/server/voice-permission-policy.ts b/packages/server/src/server/voice-permission-policy.ts index 0c0f77da0..f33b2eb16 100644 --- a/packages/server/src/server/voice-permission-policy.ts +++ b/packages/server/src/server/voice-permission-policy.ts @@ -1,7 +1,9 @@ import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js"; -const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"]; +const ALLOWED_TOKEN_SET = ["speak"]; const DENIED_TOKEN_SET = [ + "mcp", + "paseo", "bash", "shell", "terminal", @@ -27,10 +29,7 @@ function stringifyMetadata(metadata: unknown): string { } } -/** - * Voice assistant policy: allow only MCP/paseo/speak tool requests. - * All non-tool permission requests are denied. - */ +/** Voice assistant policy: only allow the speak tool. */ export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean { if (request.kind !== "tool") { return false; @@ -44,9 +43,6 @@ export function isVoicePermissionAllowed(request: AgentPermissionRequest): boole if (normalizedName === "speak") { return true; } - if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) { - return true; - } if (normalizedName !== "codextool") { return false; diff --git a/packages/server/src/server/voice-roundtrip.e2e.test.ts b/packages/server/src/server/voice-roundtrip.e2e.test.ts index fdeaa3016..40378e5e2 100644 --- a/packages/server/src/server/voice-roundtrip.e2e.test.ts +++ b/packages/server/src/server/voice-roundtrip.e2e.test.ts @@ -1,12 +1,15 @@ import { describe, test, expect, beforeAll, afterAll } from "vitest"; -import { randomUUID } from "node:crypto"; import { Readable } from "node:stream"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import pino from "pino"; import { createDaemonTestContext, type DaemonTestContext, } from "./test-utils/index.js"; +import { getFullAccessConfig, type AgentProvider } from "./daemon-e2e/agent-configs.js"; import { OpenAITTS } from "./speech/providers/openai/tts.js"; import { OpenAISTT } from "./speech/providers/openai/stt.js"; import { STTManager } from "./agent/stt-manager.js"; @@ -88,8 +91,6 @@ describe("voice roundtrip e2e", () => { voiceTts: { provider: "openai", explicit: true }, }, }, - voiceLlmProvider: "claude", - voiceLlmProviderExplicit: true, }); }, 60000); @@ -97,9 +98,10 @@ describe("voice roundtrip e2e", () => { await ctx.cleanup(); }, 60000); - speechTest( - "full roundtrip: voice input audio -> voice agent -> output audio -> transcribed output", - async () => { + for (const targetProvider of ["claude", "codex"] as const satisfies AgentProvider[]) { + speechTest( + `full roundtrip (${targetProvider}): voice input audio -> voice agent -> output audio -> transcribed output`, + async () => { const logger = pino({ level: "silent" }); const ttsProvider = new OpenAITTS( { @@ -118,7 +120,20 @@ describe("voice roundtrip e2e", () => { ); const sttOutput = new STTManager("voice-roundtrip-e2e", logger, sttProvider); - const voiceAgentId = randomUUID(); + const voiceCwd = mkdtempSync( + path.join(tmpdir(), `voice-roundtrip-agent-${targetProvider}-`) + ); + const voiceAgent = await withTimeout( + "createVoiceTargetAgent", + 30000, + ctx.client.createAgent({ + config: { + ...getFullAccessConfig(targetProvider), + cwd: voiceCwd, + }, + }) + ); + const voiceAgentId = voiceAgent.id; const voiceMode = await withTimeout( "setVoiceMode", 15000, @@ -260,6 +275,8 @@ describe("voice roundtrip e2e", () => { } finally { offStream(); offErrors(); + await ctx.client.setVoiceMode(false).catch(() => undefined); + rmSync(voiceCwd, { recursive: true, force: true }); } })(); @@ -281,7 +298,8 @@ describe("voice roundtrip e2e", () => { expect(normalized.length).toBeGreaterThan(0); expect(normalized).toMatch(/round|trip|successful/); - }, - 180000 - ); + }, + 180000 + ); + } }); diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 8700a7526..3291e49eb 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -83,10 +83,6 @@ export class VoiceAssistantWebSocketServer { }; } | null; private readonly voice: { - voiceLlmProvider?: AgentProvider | null; - voiceLlmModeId?: string | null; - voiceLlmProviderExplicit?: boolean; - voiceLlmModel?: string | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise; removeVoiceMcpSocketForAgent?: (agentId: string) => Promise; @@ -110,10 +106,6 @@ export class VoiceAssistantWebSocketServer { speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null }, terminalManager?: TerminalManager | null, voice?: { - voiceLlmProvider?: AgentProvider | null; - voiceLlmModeId?: string | null; - voiceLlmProviderExplicit?: boolean; - voiceLlmModel?: string | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise; removeVoiceMcpSocketForAgent?: (agentId: string) => Promise; diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index 15a7c8eb7..d056503d8 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -348,7 +348,7 @@ export const ArchiveAgentRequestMessageSchema = z.object({ export const SetVoiceModeMessageSchema = z.object({ type: z.literal("set_voice_mode"), enabled: z.boolean(), - voiceAgentId: z.string().optional(), + agentId: z.string().optional(), requestId: z.string().optional(), }); @@ -578,7 +578,7 @@ export const SetVoiceModeResponseMessageSchema = z.object({ payload: z.object({ requestId: z.string(), enabled: z.boolean(), - voiceAgentId: z.string().nullable(), + agentId: z.string().nullable(), accepted: z.boolean(), error: z.string().nullable(), }),