Update files

This commit is contained in:
Mohamed Boudra
2026-02-07 14:06:11 +07:00
parent cef7d8c134
commit 3cc7feab21
19 changed files with 556 additions and 630 deletions
@@ -4,10 +4,11 @@ import {
Text, Text,
ActivityIndicator, ActivityIndicator,
Platform, Platform,
Alert,
} from "react-native"; } from "react-native";
import { useState, useEffect, useRef, useCallback, useMemo } from "react"; import { useState, useEffect, useRef, useCallback, useMemo } from "react";
import { StyleSheet, useUnistyles } from "react-native-unistyles"; import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { ArrowUp, Square, Pencil } from "lucide-react-native"; import { ArrowUp, Square, Pencil, AudioLines } from "lucide-react-native";
import Animated, { useAnimatedStyle } from "react-native-reanimated"; import Animated, { useAnimatedStyle } from "react-native-reanimated";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
import { useSafeAreaInsets } from "react-native-safe-area-context"; import { useSafeAreaInsets } from "react-native-safe-area-context";
@@ -30,6 +31,7 @@ import { useAgentCommandsQuery } from "@/hooks/use-agent-commands-query";
import { encodeImages } from "@/utils/encode-images"; import { encodeImages } from "@/utils/encode-images";
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store"; import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
import { focusWithRetries } from "@/utils/web-focus"; import { focusWithRetries } from "@/utils/web-focus";
import { useVoiceOptional } from "@/contexts/voice-context";
type QueuedMessage = { type QueuedMessage = {
id: string; id: string;
@@ -76,6 +78,7 @@ export function AgentInputArea({
const client = useSessionStore( const client = useSessionStore(
(state) => state.sessions[serverId]?.client ?? null (state) => state.sessions[serverId]?.client ?? null
); );
const voice = useVoiceOptional();
const isConnected = client?.isConnected ?? false; const isConnected = client?.isConnected ?? false;
const agent = useSessionStore((state) => const agent = useSessionStore((state) =>
@@ -411,6 +414,29 @@ export function AgentInputArea({
messageInputRef.current?.focus(); messageInputRef.current?.focus();
} }
const isVoiceModeForAgent =
voice?.isVoiceModeForAgent(serverId, agentId) ?? false;
const handleToggleRealtimeVoice = useCallback(() => {
if (!voice || !isConnected) {
return;
}
if (voice.isVoiceSwitching) {
return;
}
if (voice.isVoiceModeForAgent(serverId, agentId)) {
void voice.stopVoice().catch((error) => {
console.error("[AgentInputArea] Failed to stop voice mode", error);
Alert.alert("Voice failed", "Unable to stop realtime voice mode.");
});
return;
}
void voice.startVoice(serverId, agentId).catch((error) => {
console.error("[AgentInputArea] Failed to start voice mode", error);
Alert.alert("Voice failed", "Unable to start realtime voice mode.");
});
}, [agentId, isConnected, serverId, voice]);
function handleEditQueuedMessage(id: string) { function handleEditQueuedMessage(id: string) {
const item = queuedMessages.find((q) => q.id === id); const item = queuedMessages.find((q) => q.id === id);
if (!item) return; if (!item) return;
@@ -494,7 +520,7 @@ export function AgentInputArea({
] ]
); );
const rightContent = isAgentRunning ? ( const cancelButton = isAgentRunning ? (
<Pressable <Pressable
onPress={handleCancelAgent} onPress={handleCancelAgent}
disabled={!isConnected || isCancellingAgent} disabled={!isConnected || isCancellingAgent}
@@ -515,6 +541,35 @@ export function AgentInputArea({
</Pressable> </Pressable>
) : null; ) : null;
const rightContent = (
<View style={styles.rightControls}>
<Pressable
onPress={handleToggleRealtimeVoice}
disabled={!isConnected || voice?.isVoiceSwitching}
accessibilityLabel={
isVoiceModeForAgent ? "Disable realtime voice mode" : "Enable realtime voice mode"
}
accessibilityRole="button"
style={[
styles.realtimeVoiceButton as any,
isVoiceModeForAgent ? (styles.realtimeVoiceButtonActive as any) : undefined,
(!isConnected || voice?.isVoiceSwitching
? styles.buttonDisabled
: undefined) as any,
]}
>
{voice?.isVoiceSwitching ? (
<ActivityIndicator size="small" color="white" />
) : isVoiceModeForAgent ? (
<Square size={16} color="white" fill="white" />
) : (
<AudioLines size={18} color={theme.colors.foreground} />
)}
</Pressable>
{cancelButton}
</View>
);
const leftContent = <AgentStatusBar agentId={agentId} serverId={serverId} />; const leftContent = <AgentStatusBar agentId={agentId} serverId={serverId} />;
return ( return (
@@ -588,6 +643,8 @@ export function AgentInputArea({
isScreenFocused={isScreenFocused} isScreenFocused={isScreenFocused}
leftContent={leftContent} leftContent={leftContent}
rightContent={rightContent} rightContent={rightContent}
voiceServerId={serverId}
voiceAgentId={agentId}
isAgentRunning={isAgentRunning} isAgentRunning={isAgentRunning}
onQueue={handleQueue} onQueue={handleQueue}
onKeyPress={handleCommandKeyPress} onKeyPress={handleCommandKeyPress}
@@ -631,6 +688,25 @@ const styles = StyleSheet.create(((theme: Theme) => ({
alignItems: "center", alignItems: "center",
justifyContent: "center", justifyContent: "center",
}, },
rightControls: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[2],
},
realtimeVoiceButton: {
width: 34,
height: 34,
borderRadius: theme.borderRadius.full,
backgroundColor: theme.colors.surface0,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
alignItems: "center",
justifyContent: "center",
},
realtimeVoiceButtonActive: {
backgroundColor: theme.colors.palette.green[600],
borderColor: theme.colors.palette.green[800],
},
buttonDisabled: { buttonDisabled: {
opacity: 0.5, opacity: 0.5,
}, },
@@ -102,7 +102,7 @@ export function AgentStreamView({
const flatListRef = useRef<FlatList<StreamItem>>(null); const flatListRef = useRef<FlatList<StreamItem>>(null);
const { theme } = useUnistyles(); const { theme } = useUnistyles();
const insets = useSafeAreaInsets(); const insets = useSafeAreaInsets();
const { isVoiceMode } = useVoice(); const { isVoiceModeForAgent } = useVoice();
const [isNearBottom, setIsNearBottom] = useState(true); const [isNearBottom, setIsNearBottom] = useState(true);
const hasScrolledInitially = useRef(false); const hasScrolledInitially = useRef(false);
const hasAutoScrolledOnce = useRef(false); const hasAutoScrolledOnce = useRef(false);
@@ -114,6 +114,7 @@ export function AgentStreamView({
// Get serverId (fallback to agent's serverId if not provided) // Get serverId (fallback to agent's serverId if not provided)
const resolvedServerId = serverId ?? agent.serverId ?? ""; const resolvedServerId = serverId ?? agent.serverId ?? "";
const isVoiceMode = isVoiceModeForAgent(resolvedServerId, agentId);
const client = useSessionStore( const client = useSessionStore(
(state) => state.sessions[resolvedServerId]?.client ?? null (state) => state.sessions[resolvedServerId]?.client ?? null
+12 -3
View File
@@ -61,6 +61,8 @@ export interface MessageInputProps {
leftContent?: React.ReactNode; leftContent?: React.ReactNode;
/** Content to render on the right side after voice button (e.g., realtime button, cancel button) */ /** Content to render on the right side after voice button (e.g., realtime button, cancel button) */
rightContent?: React.ReactNode; rightContent?: React.ReactNode;
voiceServerId?: string;
voiceAgentId?: string;
/** When true and there's sendable content, calls onQueue instead of onSubmit */ /** When true and there's sendable content, calls onQueue instead of onSubmit */
isAgentRunning?: boolean; isAgentRunning?: boolean;
/** Callback for queue button when agent is running */ /** Callback for queue button when agent is running */
@@ -117,6 +119,8 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
isScreenFocused = true, isScreenFocused = true,
leftContent, leftContent,
rightContent, rightContent,
voiceServerId,
voiceAgentId,
isAgentRunning = false, isAgentRunning = false,
onQueue, onQueue,
onKeyPress: onKeyPressCallback, onKeyPress: onKeyPressCallback,
@@ -310,15 +314,20 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
})); }));
const handleVoicePress = useCallback(async () => { const handleVoicePress = useCallback(async () => {
const isRealtimeVoiceForCurrentAgent =
voice &&
!!voiceServerId &&
!!voiceAgentId &&
voice.isVoiceModeForAgent(voiceServerId, voiceAgentId);
if (isDictating) { if (isDictating) {
await cancelDictation(); await cancelDictation();
} else { } else {
if (voice?.isVoiceMode) { if (isRealtimeVoiceForCurrentAgent) {
await voice.stopVoice(); await voice.stopVoice();
} }
await startDictation(); await startDictation();
} }
}, [isDictating, cancelDictation, startDictation, voice]); }, [isDictating, cancelDictation, startDictation, voice, voiceAgentId, voiceServerId]);
const handleCancelRecording = useCallback(async () => { const handleCancelRecording = useCallback(async () => {
await cancelDictation(); await cancelDictation();
@@ -626,6 +635,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
<Mic size={20} color={theme.colors.foreground} /> <Mic size={20} color={theme.colors.foreground} />
)} )}
</Pressable> </Pressable>
{rightContent}
{shouldShowSendButton && ( {shouldShowSendButton && (
<Pressable <Pressable
onPress={handleSendMessage} onPress={handleSendMessage}
@@ -647,7 +657,6 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
<ArrowUp size={20} color="white" /> <ArrowUp size={20} color="white" />
</Pressable> </Pressable>
)} )}
{!shouldShowSendButton && rightContent}
</View> </View>
</View> </View>
</Animated.View> </Animated.View>
+2 -198
View File
@@ -1,5 +1,5 @@
import { useCallback, useMemo, useState, useEffect } from "react"; import { useCallback, useMemo, useState, useEffect } from "react";
import { View, Pressable, Text, Platform, Alert, ActivityIndicator } from "react-native"; import { View, Pressable, Text, Platform } from "react-native";
import { useSafeAreaInsets } from "react-native-safe-area-context"; import { useSafeAreaInsets } from "react-native-safe-area-context";
import Animated, { import Animated, {
useAnimatedStyle, useAnimatedStyle,
@@ -9,21 +9,17 @@ import Animated, {
} from "react-native-reanimated"; } from "react-native-reanimated";
import { Gesture, GestureDetector } from "react-native-gesture-handler"; import { Gesture, GestureDetector } from "react-native-gesture-handler";
import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles"; import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles";
import { Plus, Settings, Users, AudioLines } from "lucide-react-native"; import { Plus, Settings, Users } from "lucide-react-native";
import { router } from "expo-router"; import { router } from "expo-router";
import { usePanelStore } from "@/stores/panel-store"; import { usePanelStore } from "@/stores/panel-store";
import { GroupedAgentList } from "./grouped-agent-list"; import { GroupedAgentList } from "./grouped-agent-list";
import { useAggregatedAgents } from "@/hooks/use-aggregated-agents"; import { useAggregatedAgents } from "@/hooks/use-aggregated-agents";
import { useSidebarAnimation } from "@/contexts/sidebar-animation-context"; import { useSidebarAnimation } from "@/contexts/sidebar-animation-context";
import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window"; import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window";
import { useVoice } from "@/contexts/voice-context";
import { useDaemonConnections } from "@/contexts/daemon-connections-context";
import { VoicePanel } from "./voice-panel";
import { useSidebarAgentSections } from "@/hooks/use-sidebar-agent-sections"; import { useSidebarAgentSections } from "@/hooks/use-sidebar-agent-sections";
import { useSidebarCollapsedSectionsStore } from "@/stores/sidebar-collapsed-sections-store"; import { useSidebarCollapsedSectionsStore } from "@/stores/sidebar-collapsed-sections-store";
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store"; import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
import { deriveSidebarShortcutAgentKeys } from "@/utils/sidebar-shortcuts"; import { deriveSidebarShortcutAgentKeys } from "@/utils/sidebar-shortcuts";
import { AdaptiveModalSheet } from "@/components/adaptive-modal-sheet";
const DESKTOP_SIDEBAR_WIDTH = 320; const DESKTOP_SIDEBAR_WIDTH = 320;
const SIDEBAR_AGENT_LIMIT = 50; // temporary for screenshots const SIDEBAR_AGENT_LIMIT = 50; // temporary for screenshots
@@ -56,9 +52,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
} = useSidebarAnimation(); } = useSidebarAnimation();
const trafficLightPadding = useTrafficLightPadding(); const trafficLightPadding = useTrafficLightPadding();
const dragHandlers = useTauriDragHandlers(); const dragHandlers = useTauriDragHandlers();
const { connectionStates } = useDaemonConnections();
const { isVoiceMode, isVoiceSwitching, startVoice, stopVoice } = useVoice();
const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false);
// Track user-initiated refresh to avoid showing spinner on background revalidation // Track user-initiated refresh to avoid showing spinner on background revalidation
const [isManualRefresh, setIsManualRefresh] = useState(false); const [isManualRefresh, setIsManualRefresh] = useState(false);
@@ -142,65 +135,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
router.push("/agents"); router.push("/agents");
}, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]); }, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]);
const voiceEligibleHosts = useMemo(() => {
return Array.from(connectionStates.values()).filter((entry) => entry.status === "online");
}, [connectionStates]);
const hasAnyConfiguredHosts = connectionStates.size > 0;
const handleToggleVoice = useCallback(() => {
if (isVoiceSwitching) {
return;
}
if (isVoiceMode) {
void stopVoice().catch((error) => {
console.error("[SlidingSidebar] Failed to stop voice", error);
Alert.alert("Voice failed", "Unable to stop voice mode.");
});
return;
}
if (voiceEligibleHosts.length === 0) {
if (!hasAnyConfiguredHosts) {
Alert.alert(
"No hosts available",
"Add a host in Settings before starting Voice mode.",
[{ text: "Open Settings", onPress: () => router.push("/settings") }, { text: "OK" }]
);
return;
}
Alert.alert(
"Hosts reconnecting",
"Every host is offline right now. Paseo reconnects automatically—try Voice mode again once one comes online."
);
return;
}
if (voiceEligibleHosts.length === 1) {
void startVoice(voiceEligibleHosts[0].daemon.serverId).catch((error) => {
console.error("[SlidingSidebar] Failed to start voice", error);
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
});
return;
}
setShowVoiceHostPicker(true);
}, [hasAnyConfiguredHosts, isVoiceMode, isVoiceSwitching, startVoice, stopVoice, voiceEligibleHosts]);
const handleSelectVoiceHost = useCallback(
(serverId: string) => {
setShowVoiceHostPicker(false);
void startVoice(serverId).catch((error) => {
console.error("[SlidingSidebar] Failed to start voice", error);
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
});
},
[startVoice]
);
const handleDismissVoiceHostPicker = useCallback(() => {
setShowVoiceHostPicker(false);
}, []);
// Close gesture (swipe left to close when sidebar is open) // Close gesture (swipe left to close when sidebar is open)
// Only activates on leftward swipe, fails on rightward or vertical movement // Only activates on leftward swipe, fails on rightward or vertical movement
// This mirrors the explorer-sidebar pattern for the right sidebar // This mirrors the explorer-sidebar pattern for the right sidebar
@@ -301,8 +235,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
parentGestureRef={closeGestureRef} parentGestureRef={closeGestureRef}
/> />
{isVoiceMode ? <VoicePanel /> : null}
{/* Footer */} {/* Footer */}
<View style={styles.sidebarFooter}> <View style={styles.sidebarFooter}>
<Pressable <Pressable
@@ -319,43 +251,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
)} )}
</Pressable> </Pressable>
<View style={styles.footerIconRow}> <View style={styles.footerIconRow}>
<Pressable
style={styles.footerIconButton}
testID="sidebar-voice"
nativeID="sidebar-voice"
collapsable={false}
accessible
accessibilityLabel="Voice"
accessibilityRole="button"
disabled={isVoiceSwitching}
onPress={handleToggleVoice}
>
{({ hovered }) => (
isVoiceSwitching ? (
<ActivityIndicator
size="small"
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
) : (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)
)}
</Pressable>
<Pressable <Pressable
style={styles.footerIconButton} style={styles.footerIconButton}
testID="sidebar-settings" testID="sidebar-settings"
@@ -375,32 +270,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
</View> </View>
</Animated.View> </Animated.View>
</GestureDetector> </GestureDetector>
<AdaptiveModalSheet
title="Choose a host"
visible={showVoiceHostPicker}
onClose={handleDismissVoiceHostPicker}
snapPoints={["45%", "70%"]}
testID="voice-host-picker"
>
<View style={styles.hostPickerList}>
{voiceEligibleHosts.map((entry) => (
<Pressable
key={entry.daemon.serverId}
style={styles.hostPickerOption}
onPress={() => handleSelectVoiceHost(entry.daemon.serverId)}
>
<Text style={styles.hostPickerOptionText}>{entry.daemon.label}</Text>
</Pressable>
))}
<Pressable
style={styles.hostPickerCancel}
onPress={handleDismissVoiceHostPicker}
>
<Text style={styles.hostPickerCancelText}>Cancel</Text>
</Pressable>
</View>
</AdaptiveModalSheet>
</View> </View>
); );
} }
@@ -439,8 +308,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
selectedAgentId={selectedAgentId} selectedAgentId={selectedAgentId}
/> />
{isVoiceMode ? <VoicePanel /> : null}
{/* Footer */} {/* Footer */}
<View style={styles.sidebarFooter}> <View style={styles.sidebarFooter}>
<Pressable <Pressable
@@ -457,43 +324,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
)} )}
</Pressable> </Pressable>
<View style={styles.footerIconRow}> <View style={styles.footerIconRow}>
<Pressable
style={styles.footerIconButton}
testID="sidebar-voice"
nativeID="sidebar-voice"
collapsable={false}
accessible
accessibilityLabel="Voice"
accessibilityRole="button"
disabled={isVoiceSwitching}
onPress={handleToggleVoice}
>
{({ hovered }) => (
isVoiceSwitching ? (
<ActivityIndicator
size="small"
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
) : (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)
)}
</Pressable>
<Pressable <Pressable
style={styles.footerIconButton} style={styles.footerIconButton}
testID="sidebar-settings" testID="sidebar-settings"
@@ -510,32 +340,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
</Pressable> </Pressable>
</View> </View>
</View> </View>
<AdaptiveModalSheet
title="Choose a host"
visible={showVoiceHostPicker}
onClose={handleDismissVoiceHostPicker}
snapPoints={["45%", "70%"]}
testID="voice-host-picker"
>
<View style={styles.hostPickerList}>
{voiceEligibleHosts.map((entry) => (
<Pressable
key={entry.daemon.serverId}
style={styles.hostPickerOption}
onPress={() => handleSelectVoiceHost(entry.daemon.serverId)}
>
<Text style={styles.hostPickerOptionText}>{entry.daemon.label}</Text>
</Pressable>
))}
<Pressable
style={styles.hostPickerCancel}
onPress={handleDismissVoiceHostPicker}
>
<Text style={styles.hostPickerCancelText}>Cancel</Text>
</Pressable>
</View>
</AdaptiveModalSheet>
</View> </View>
); );
} }
+30 -7
View File
@@ -18,10 +18,12 @@ interface VoiceContextValue {
isDetecting: boolean; isDetecting: boolean;
isSpeaking: boolean; isSpeaking: boolean;
segmentDuration: number; segmentDuration: number;
startVoice: (serverId: string) => Promise<void>; startVoice: (serverId: string, agentId: string) => Promise<void>;
stopVoice: () => Promise<void>; stopVoice: () => Promise<void>;
isVoiceModeForAgent: (serverId: string, agentId: string) => boolean;
toggleMute: () => void; toggleMute: () => void;
activeServerId: string | null; activeServerId: string | null;
activeAgentId: string | null;
} }
const VoiceContext = createContext<VoiceContextValue | null>(null); const VoiceContext = createContext<VoiceContextValue | null>(null);
@@ -45,6 +47,7 @@ interface VoiceProviderProps {
export function VoiceProvider({ children }: VoiceProviderProps) { export function VoiceProvider({ children }: VoiceProviderProps) {
const getSession = useSessionStore((state) => state.getSession); const getSession = useSessionStore((state) => state.getSession);
const [activeServerId, setActiveServerId] = useState<string | null>(null); const [activeServerId, setActiveServerId] = useState<string | null>(null);
const [activeAgentId, setActiveAgentId] = useState<string | null>(null);
const activeSession = useSessionStore( const activeSession = useSessionStore(
useCallback( useCallback(
(state: ReturnType<typeof useSessionStore.getState>) => { (state: ReturnType<typeof useSessionStore.getState>) => {
@@ -151,7 +154,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
voiceTransportReadyRef.current = false; voiceTransportReadyRef.current = false;
} }
if (!isVoiceMode || !activeServerId || !client) { if (!isVoiceMode || !activeServerId || !activeAgentId || !client) {
wasVoiceSocketConnectedRef.current = connected; wasVoiceSocketConnectedRef.current = connected;
if (!isVoiceMode) { if (!isVoiceMode) {
lastVoiceModeSyncedClientRef.current = null; lastVoiceModeSyncedClientRef.current = null;
@@ -167,7 +170,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
voiceResyncInFlightRef.current = true; voiceResyncInFlightRef.current = true;
voiceTransportReadyRef.current = false; voiceTransportReadyRef.current = false;
setIsVoiceSwitching(true); setIsVoiceSwitching(true);
void client.setVoiceMode(true).then( void client.setVoiceMode(true, activeAgentId).then(
() => { () => {
console.log("[Voice] Re-synced voice mode after reconnect"); console.log("[Voice] Re-synced voice mode after reconnect");
lastVoiceModeSyncedClientRef.current = client; lastVoiceModeSyncedClientRef.current = client;
@@ -184,7 +187,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
} }
wasVoiceSocketConnectedRef.current = connected; wasVoiceSocketConnectedRef.current = connected;
}, [activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]); }, [activeAgentId, activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]);
const isPlayingAudio = activeSession?.isPlayingAudio ?? false; const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
@@ -201,7 +204,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
}, [isPlayingAudio]); }, [isPlayingAudio]);
const startVoice = useCallback( const startVoice = useCallback(
async (serverId: string) => { async (serverId: string, agentId: string) => {
const session = getSession(serverId) ?? null; const session = getSession(serverId) ?? null;
if (!session) { if (!session) {
throw new Error(`Host ${serverId} is not connected`); throw new Error(`Host ${serverId} is not connected`);
@@ -210,13 +213,23 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
setIsVoiceSwitching(true); setIsVoiceSwitching(true);
voiceTransportReadyRef.current = false; voiceTransportReadyRef.current = false;
try { try {
const previousSession = realtimeSessionRef.current;
if (
isVoiceMode &&
previousSession?.client &&
(activeServerId !== serverId || activeAgentId !== agentId)
) {
await previousSession.client.setVoiceMode(false);
}
realtimeSessionRef.current = session; realtimeSessionRef.current = session;
setActiveServerId(serverId); setActiveServerId(serverId);
setActiveAgentId(agentId);
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => { await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
console.warn("[Voice] Failed to activate keep-awake:", error); console.warn("[Voice] Failed to activate keep-awake:", error);
}); });
if (session?.client) { if (session?.client) {
await session.client.setVoiceMode(true); await session.client.setVoiceMode(true, agentId);
} else { } else {
console.warn("[Voice] setVoiceMode skipped: daemon unavailable"); console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
} }
@@ -230,13 +243,14 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
console.error("[Voice] Failed to start:", error); console.error("[Voice] Failed to start:", error);
await realtimeAudio.stop().catch(() => undefined); await realtimeAudio.stop().catch(() => undefined);
setActiveServerId((current) => (current === serverId ? null : current)); setActiveServerId((current) => (current === serverId ? null : current));
setActiveAgentId((current) => (current === agentId ? null : current));
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined); await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
throw error; throw error;
} finally { } finally {
setIsVoiceSwitching(false); setIsVoiceSwitching(false);
} }
}, },
[getSession, realtimeAudio] [activeAgentId, activeServerId, getSession, isVoiceMode, realtimeAudio]
); );
const stopVoice = useCallback(async () => { const stopVoice = useCallback(async () => {
@@ -254,6 +268,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
await realtimeAudio.stop(); await realtimeAudio.stop();
setIsVoiceMode(false); setIsVoiceMode(false);
setActiveServerId(null); setActiveServerId(null);
setActiveAgentId(null);
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined); await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
console.log("[Voice] Mode disabled"); console.log("[Voice] Mode disabled");
} catch (error: any) { } catch (error: any) {
@@ -265,6 +280,12 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
} }
}, [realtimeAudio]); }, [realtimeAudio]);
const isVoiceModeForAgent = useCallback(
(serverId: string, agentId: string) =>
isVoiceMode && activeServerId === serverId && activeAgentId === agentId,
[activeAgentId, activeServerId, isVoiceMode]
);
const value: VoiceContextValue = { const value: VoiceContextValue = {
isVoiceMode, isVoiceMode,
isVoiceSwitching, isVoiceSwitching,
@@ -275,8 +296,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
segmentDuration: realtimeAudio.segmentDuration, segmentDuration: realtimeAudio.segmentDuration,
startVoice, startVoice,
stopVoice, stopVoice,
isVoiceModeForAgent,
toggleMute: realtimeAudio.toggleMute, toggleMute: realtimeAudio.toggleMute,
activeServerId, activeServerId,
activeAgentId,
}; };
return ( return (
@@ -1,4 +1,6 @@
import { randomUUID } from "node:crypto"; import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import pino from "pino"; import pino from "pino";
import { createTestPaseoDaemon } from "../src/server/test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../src/server/test-utils/paseo-daemon.js";
@@ -48,9 +50,20 @@ async function main(): Promise<void> {
await client.connect(); await client.connect();
client.subscribeAgentUpdates({ subscriptionId: "voice-debug" }); client.subscribeAgentUpdates({ subscriptionId: "voice-debug" });
const voiceAgentId = randomUUID(); const voiceCwd = mkdtempSync(path.join(tmpdir(), "voice-roundtrip-debug-"));
const voiceAgent = await client.createAgent({
config: {
provider: "claude",
cwd: voiceCwd,
modeId: "bypassPermissions",
},
});
const voiceAgentId = voiceAgent.id;
const mode = await client.setVoiceMode(true, voiceAgentId); const mode = await client.setVoiceMode(true, voiceAgentId);
console.log("set_voice_mode_response", mode); console.log("set_voice_mode_response", mode);
if (!mode.accepted) {
throw new Error(`setVoiceMode rejected: ${mode.error ?? "unknown error"}`);
}
const offTranscript = client.on("transcription_result", (msg) => { const offTranscript = client.on("transcription_result", (msg) => {
if (msg.type !== "transcription_result") return; if (msg.type !== "transcription_result") return;
@@ -149,6 +162,7 @@ async function main(): Promise<void> {
console.log("success", { audioChunkCount }); console.log("success", { audioChunkCount });
await client.setVoiceMode(false); await client.setVoiceMode(false);
rmSync(voiceCwd, { recursive: true, force: true });
offTranscript(); offTranscript();
offActivity(); offActivity();
+2 -2
View File
@@ -1291,13 +1291,13 @@ export class DaemonClient {
async setVoiceMode( async setVoiceMode(
enabled: boolean, enabled: boolean,
voiceAgentId?: string agentId?: string
): Promise<SetVoiceModePayload> { ): Promise<SetVoiceModePayload> {
const requestId = this.createRequestId(); const requestId = this.createRequestId();
const message = SessionInboundMessageSchema.parse({ const message = SessionInboundMessageSchema.parse({
type: "set_voice_mode", type: "set_voice_mode",
enabled, enabled,
...(voiceAgentId ? { voiceAgentId } : {}), ...(agentId ? { agentId } : {}),
requestId, requestId,
}); });
return this.sendRequest({ return this.sendRequest({
@@ -407,22 +407,24 @@ export class AgentManager {
); );
} }
async refreshAgentFromPersistence(agentId: string): Promise<ManagedAgent> { async refreshAgentFromPersistence(
agentId: string,
overrides?: Partial<AgentSessionConfig>
): Promise<ManagedAgent> {
const existing = this.requireAgent(agentId); const existing = this.requireAgent(agentId);
const handle = existing.persistence; const handle = existing.persistence;
if (!handle) { const provider = handle?.provider ?? existing.provider;
throw new Error( const client = this.requireClient(provider);
`Agent ${agentId} cannot be refreshed because it has no persistence handle` const refreshConfig = {
);
}
const client = this.requireClient(handle.provider);
const overrides = {
...existing.config, ...existing.config,
provider: handle.provider, ...overrides,
}; provider,
} as AgentSessionConfig;
const normalizedConfig = await this.normalizeConfig(refreshConfig);
const session = await client.resumeSession(handle, overrides); const session = handle
? await client.resumeSession(handle, normalizedConfig)
: await client.createSession(normalizedConfig);
// Remove the existing agent entry before swapping sessions // Remove the existing agent entry before swapping sessions
this.agents.delete(agentId); this.agents.delete(agentId);
@@ -436,7 +438,12 @@ export class AgentManager {
} }
// Preserve existing labels during refresh // Preserve existing labels during refresh
return this.registerSession(session, overrides, agentId, { labels: existing.labels }); return this.registerSession(session, normalizedConfig, agentId, {
labels: existing.labels,
createdAt: existing.createdAt,
updatedAt: existing.updatedAt,
lastUserMessageAt: existing.lastUserMessageAt,
});
} }
async closeAgent(agentId: string): Promise<void> { async closeAgent(agentId: string): Promise<void> {
@@ -57,6 +57,7 @@ export interface AgentMcpServerOptions {
callerAgentId: string callerAgentId: string
) => VoiceCallerContext | null; ) => VoiceCallerContext | null;
enableVoiceTools?: boolean; enableVoiceTools?: boolean;
voiceOnly?: boolean;
logger: Logger; logger: Logger;
} }
@@ -393,7 +394,7 @@ export async function createAgentMcpServer(
initialMode: topLevelInputSchema.initialMode.optional(), initialMode: topLevelInputSchema.initialMode.optional(),
}); });
if (options.enableVoiceTools || callerContext?.enableVoiceTools) { if (options.voiceOnly || options.enableVoiceTools || callerContext?.enableVoiceTools) {
server.registerTool( server.registerTool(
"speak", "speak",
{ {
@@ -432,6 +433,10 @@ export async function createAgentMcpServer(
); );
} }
if (options.voiceOnly) {
return server;
}
server.registerTool( server.registerTool(
"create_agent", "create_agent",
{ {
+2 -97
View File
@@ -64,7 +64,6 @@ import type {
AgentClient, AgentClient,
AgentProvider, AgentProvider,
} from "./agent/agent-sdk-types.js"; } from "./agent/agent-sdk-types.js";
import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js";
import { acquirePidLock, releasePidLock } from "./pid-lock.js"; import { acquirePidLock, releasePidLock } from "./pid-lock.js";
import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js"; import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js";
import { import {
@@ -278,98 +277,8 @@ export async function createPaseoDaemon(
logger.info( logger.info(
`Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand` `Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`
); );
const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null;
const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false;
const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS
.filter((definition) => definition.voice?.enabled)
.map((definition) => definition.id as AgentProvider);
logger.info( logger.info(
{ "Voice mode configured for agent-scoped resume flow (no dedicated voice assistant provider)"
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
voiceEnabledProviders,
},
"Voice LLM provider reconciliation started"
);
const providerClients = createAllClients(logger);
Object.assign(providerClients, config.agentClients);
const voiceLlmAvailability = Object.fromEntries(
voiceEnabledProviders.map((provider) => [provider, false])
) as Record<AgentProvider, boolean>;
for (const provider of voiceEnabledProviders) {
try {
voiceLlmAvailability[provider] = await providerClients[provider].isAvailable();
} catch (error) {
logger.warn({ err: error, provider }, "Voice LLM provider availability check failed");
voiceLlmAvailability[provider] = false;
}
}
let resolvedVoiceLlmProvider: AgentProvider | null = null;
if (requestedVoiceLlmProvider) {
if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) {
logger.error(
{ provider: requestedVoiceLlmProvider, voiceEnabledProviders },
"Configured voice LLM provider does not support voice mode"
);
throw new Error(
`Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode`
);
}
if (!voiceLlmAvailability[requestedVoiceLlmProvider]) {
logger.error(
{ provider: requestedVoiceLlmProvider, voiceLlmAvailability },
"Configured voice LLM provider is unavailable"
);
throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`);
}
resolvedVoiceLlmProvider = requestedVoiceLlmProvider;
} else {
resolvedVoiceLlmProvider =
voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null;
}
let resolvedVoiceLlmModeId: string | null = null;
let resolvedVoiceLlmModel: string | null = null;
if (!resolvedVoiceLlmProvider) {
if (voiceLlmProviderExplicit) {
logger.error(
{ requestedVoiceLlmProvider, voiceLlmAvailability },
"No voice LLM provider available"
);
throw new Error("No voice LLM provider available");
}
logger.warn(
{ requestedVoiceLlmProvider, voiceLlmAvailability },
"No default voice LLM provider available; voice mode will be disabled until a provider is configured"
);
} else {
const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
(definition) => definition.id === resolvedVoiceLlmProvider
);
if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
throw new Error(
`Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
);
}
resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
resolvedVoiceLlmModel =
config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;
}
logger.info(
{
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
resolvedVoiceLlmProvider,
resolvedVoiceLlmModeId,
resolvedVoiceLlmModel,
voiceLlmAvailability,
},
"Voice LLM provider reconciliation completed"
); );
let wsServer: VoiceAssistantWebSocketServer | null = null; let wsServer: VoiceAssistantWebSocketServer | null = null;
let voiceMcpBridgeManager: VoiceMcpSocketBridgeManager | null = null; let voiceMcpBridgeManager: VoiceMcpSocketBridgeManager | null = null;
@@ -523,7 +432,7 @@ export async function createPaseoDaemon(
agentStorage, agentStorage,
paseoHome: config.paseoHome, paseoHome: config.paseoHome,
callerAgentId, callerAgentId,
enableVoiceTools: false, voiceOnly: true,
resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null, resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null, resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
logger, logger,
@@ -555,10 +464,6 @@ export async function createPaseoDaemon(
{ stt: sttService, tts: ttsService }, { stt: sttService, tts: ttsService },
terminalManager, terminalManager,
{ {
voiceLlmProvider: resolvedVoiceLlmProvider,
voiceLlmModeId: resolvedVoiceLlmModeId,
voiceLlmProviderExplicit,
voiceLlmModel: resolvedVoiceLlmModel,
voiceAgentMcpStdio: { voiceAgentMcpStdio: {
command: voiceMcpBridgeCommand.command, command: voiceMcpBridgeCommand.command,
baseArgs: [...voiceMcpBridgeCommand.baseArgs], baseArgs: [...voiceMcpBridgeCommand.baseArgs],
@@ -142,23 +142,29 @@ describe("daemon client E2E", () => {
const agents = await ctx.client.fetchAgents(); const agents = await ctx.client.fetchAgents();
expect(Array.isArray(agents)).toBe(true); expect(Array.isArray(agents)).toBe(true);
const voiceAgents = await ctx.client.fetchAgents({ const cwd = tmpCwd();
filter: { labels: { surface: "voice" } }, const created = await ctx.client.createAgent({
config: {
...getFullAccessConfig("codex"),
cwd,
},
}); });
expect(Array.isArray(voiceAgents)).toBe(true);
await expect(ctx.client.setVoiceMode(true)).resolves.toMatchObject({ await expect(ctx.client.setVoiceMode(true, created.id)).resolves.toMatchObject({
enabled: true, enabled: true,
agentId: created.id,
accepted: true, accepted: true,
error: null, error: null,
}); });
await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({ await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({
enabled: false, enabled: false,
agentId: null,
accepted: true, accepted: true,
error: null, error: null,
}); });
await ctx.client.deleteAgent(randomUUID()); await ctx.client.deleteAgent(randomUUID());
rmSync(cwd, { recursive: true, force: true });
}, 30000); }, 30000);
test("emits server_info on websocket connect", async () => { test("emits server_info on websocket connect", async () => {
@@ -655,7 +661,14 @@ describe("daemon client E2E", () => {
speechTest( speechTest(
"voice mode buffers audio until isLast and emits transcription_result", "voice mode buffers audio until isLast and emits transcription_result",
async () => { async () => {
await ctx.client.setVoiceMode(true, randomUUID()); const voiceCwd = tmpCwd();
const voiceAgent = await ctx.client.createAgent({
config: {
...getFullAccessConfig("codex"),
cwd: voiceCwd,
},
});
await ctx.client.setVoiceMode(true, voiceAgent.id);
const transcription = waitForSignal(30_000, (resolve) => { const transcription = waitForSignal(30_000, (resolve) => {
const unsubscribe = ctx.client.on("transcription_result", (message) => { const unsubscribe = ctx.client.on("transcription_result", (message) => {
@@ -751,6 +764,7 @@ describe("daemon client E2E", () => {
} finally { } finally {
await Promise.allSettled([transcription, errorSignal]); await Promise.allSettled([transcription, errorSignal]);
await ctx.client.setVoiceMode(false); await ctx.client.setVoiceMode(false);
rmSync(voiceCwd, { recursive: true, force: true });
} }
}, },
90_000 90_000
+226 -210
View File
@@ -1,5 +1,5 @@
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
import { mkdir, stat } from "fs/promises"; import { stat } from "fs/promises";
import { exec } from "child_process"; import { exec } from "child_process";
import { promisify } from "util"; import { promisify } from "util";
import { join, resolve, sep } from "path"; import { join, resolve, sep } from "path";
@@ -58,6 +58,7 @@ import type {
AgentPermissionResponse, AgentPermissionResponse,
AgentPromptContentBlock, AgentPromptContentBlock,
AgentPromptInput, AgentPromptInput,
McpServerConfig,
AgentSessionConfig, AgentSessionConfig,
AgentStreamEvent, AgentStreamEvent,
AgentProvider, AgentProvider,
@@ -191,7 +192,18 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
); );
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/; const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
const VoiceAgentIdSchema = z.string().uuid(); const AgentIdSchema = z.string().uuid();
const VOICE_MCP_SERVER_NAME = "paseo_voice";
const VOICE_MODE_ENFORCED_MODE_ID: Partial<Record<AgentProvider, string>> = {
claude: "default",
codex: "read-only",
};
type VoiceModeBaseConfig = {
systemPrompt?: string;
mcpServers?: Record<string, McpServerConfig>;
modeId?: string;
};
interface AudioBufferState { interface AudioBufferState {
chunks: Buffer[]; chunks: Buffer[];
@@ -214,10 +226,6 @@ export type SessionOptions = {
tts: TextToSpeechProvider | null; tts: TextToSpeechProvider | null;
terminalManager: TerminalManager | null; terminalManager: TerminalManager | null;
voice?: { voice?: {
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
}; };
voiceBridge?: { voiceBridge?: {
@@ -376,10 +384,6 @@ export class Session {
} | null = null; } | null = null;
private readonly terminalManager: TerminalManager | null; private readonly terminalManager: TerminalManager | null;
private terminalSubscriptions: Map<string, () => void> = new Map(); private terminalSubscriptions: Map<string, () => void> = new Map();
private readonly voiceLlmProvider: AgentProvider | null;
private readonly voiceLlmModeId: string | null;
private readonly voiceLlmProviderExplicit: boolean;
private readonly voiceLlmModel: string | null;
private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null; private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null;
private readonly localSpeechModelsDir: string; private readonly localSpeechModelsDir: string;
private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[]; private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[];
@@ -395,7 +399,8 @@ export class Session {
private readonly unregisterVoiceCallerContext?: (agentId: string) => void; private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
private readonly ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>; private readonly ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
private readonly removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>; private readonly removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
private voiceAssistantAgentId: string | null = null; private voiceModeAgentId: string | null = null;
private voiceModeBaseConfig: VoiceModeBaseConfig | null = null;
constructor(options: SessionOptions) { constructor(options: SessionOptions) {
const { const {
@@ -425,10 +430,6 @@ export class Session {
this.agentStorage = agentStorage; this.agentStorage = agentStorage;
this.createAgentMcpTransport = createAgentMcpTransport; this.createAgentMcpTransport = createAgentMcpTransport;
this.terminalManager = terminalManager; this.terminalManager = terminalManager;
this.voiceLlmProvider = voice?.voiceLlmProvider ?? null;
this.voiceLlmModeId = voice?.voiceLlmModeId ?? null;
this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false;
this.voiceLlmModel = voice?.voiceLlmModel ?? null;
this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null; this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null;
const configuredModelsDir = dictation?.localModels?.modelsDir?.trim(); const configuredModelsDir = dictation?.localModels?.modelsDir?.trim();
this.localSpeechModelsDir = this.localSpeechModelsDir =
@@ -896,7 +897,7 @@ export class Session {
break; break;
case "set_voice_mode": case "set_voice_mode":
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId, msg.requestId); await this.handleSetVoiceMode(msg.enabled, msg.agentId, msg.requestId);
break; break;
case "send_agent_message_request": case "send_agent_message_request":
@@ -1276,108 +1277,216 @@ export class Session {
*/ */
private async handleSetVoiceMode( private async handleSetVoiceMode(
enabled: boolean, enabled: boolean,
voiceAgentId?: string, agentId?: string,
requestId?: string requestId?: string
): Promise<void> { ): Promise<void> {
if (enabled) { try {
let normalizedVoiceAgentId: string | null = null; if (enabled) {
if (voiceAgentId && voiceAgentId.trim().length > 0) { const normalizedAgentId = this.parseVoiceTargetAgentId(
normalizedVoiceAgentId = this.parseVoiceAgentId( agentId ?? "",
voiceAgentId,
"set_voice_mode" "set_voice_mode"
); );
} else {
normalizedVoiceAgentId = await this.findExistingVoiceAssistantAgentId(); if (
this.isVoiceMode &&
this.voiceModeAgentId &&
this.voiceModeAgentId !== normalizedAgentId
) {
await this.disableVoiceModeForActiveAgent(true);
}
if (!this.isVoiceMode || this.voiceModeAgentId !== normalizedAgentId) {
const refreshedAgentId = await this.enableVoiceModeForAgent(normalizedAgentId);
this.voiceModeAgentId = refreshedAgentId;
}
this.isVoiceMode = true;
this.sessionLogger.info(
{
agentId: this.voiceModeAgentId,
},
"Voice mode enabled for existing agent"
);
if (requestId) {
this.emit({
type: "set_voice_mode_response",
payload: {
requestId,
enabled: true,
agentId: this.voiceModeAgentId,
accepted: true,
error: null,
},
});
}
return;
} }
this.isVoiceMode = true; await this.disableVoiceModeForActiveAgent(true);
this.voiceAssistantAgentId = normalizedVoiceAgentId; this.isVoiceMode = false;
this.sessionLogger.info( this.sessionLogger.info("Voice mode disabled");
if (requestId) {
this.emit({
type: "set_voice_mode_response",
payload: {
requestId,
enabled: false,
agentId: null,
accepted: true,
error: null,
},
});
}
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to set voice mode";
this.sessionLogger.error(
{ {
voiceAssistantAgentId: this.voiceAssistantAgentId, err: error,
resumedVoiceAgent: Boolean(normalizedVoiceAgentId), enabled,
providedVoiceAgentId: Boolean(voiceAgentId?.trim()), requestedAgentId: agentId ?? null,
}, },
"Voice mode enabled (agent-backed)" "set_voice_mode failed"
); );
if (requestId) { if (requestId) {
this.emit({ this.emit({
type: "set_voice_mode_response", type: "set_voice_mode_response",
payload: { payload: {
requestId, requestId,
enabled: true, enabled: this.isVoiceMode,
voiceAgentId: this.voiceAssistantAgentId, agentId: this.voiceModeAgentId,
accepted: true, accepted: false,
error: null, error: errorMessage,
}, },
}); });
return;
} }
return; throw error;
}
this.isVoiceMode = false;
const disabledVoiceAgentId = this.voiceAssistantAgentId;
if (disabledVoiceAgentId && this.removeVoiceMcpSocketForAgent) {
try {
await this.removeVoiceMcpSocketForAgent(disabledVoiceAgentId);
} catch (error) {
this.sessionLogger.warn(
{ err: error, voiceAssistantAgentId: disabledVoiceAgentId },
"Failed to remove voice MCP socket bridge on disable"
);
}
}
this.sessionLogger.info(
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
"Voice mode disabled (agent-backed)"
);
if (requestId) {
this.emit({
type: "set_voice_mode_response",
payload: {
requestId,
enabled: false,
voiceAgentId: this.voiceAssistantAgentId,
accepted: true,
error: null,
},
});
} }
} }
private parseVoiceAgentId(rawId: string, source: string): string { private parseVoiceTargetAgentId(rawId: string, source: string): string {
const parsed = VoiceAgentIdSchema.safeParse(rawId.trim()); const parsed = AgentIdSchema.safeParse(rawId.trim());
if (!parsed.success) { if (!parsed.success) {
throw new Error(`${source}: voiceAgentId must be a UUID`); throw new Error(`${source}: agentId must be a UUID`);
} }
return parsed.data; return parsed.data;
} }
private async findExistingVoiceAssistantAgentId(): Promise<string | null> { private cloneMcpServers(
const scopedCandidates = await this.listAgentPayloads({ servers: Record<string, McpServerConfig> | undefined
labels: { ): Record<string, McpServerConfig> | undefined {
surface: "voice", if (!servers) {
voiceClientId: this.clientId, return undefined;
},
});
const fallbackCandidates = await this.listAgentPayloads({
labels: { surface: "voice" },
});
const available = [
...scopedCandidates,
...fallbackCandidates.filter(
(candidate) => !scopedCandidates.some((scoped) => scoped.id === candidate.id)
),
].filter((agent) => !agent.archivedAt && agent.labels?.ui !== "false");
if (available.length === 0) {
return null;
} }
available.sort((left, right) => { return JSON.parse(JSON.stringify(servers)) as Record<string, McpServerConfig>;
const leftTime = Date.parse(left.updatedAt); }
const rightTime = Date.parse(right.updatedAt);
return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0); private buildVoiceModeSystemPrompt(existing?: string): string {
const chunks = [existing?.trim(), VOICE_AGENT_SYSTEM_INSTRUCTION].filter(
(value): value is string => Boolean(value && value.length > 0)
);
return chunks.join("\n\n");
}
private buildVoiceModeMcpServers(
existing: Record<string, McpServerConfig> | undefined,
socketPath: string
): Record<string, McpServerConfig> {
const mcpStdio = this.voiceAgentMcpStdio;
if (!mcpStdio) {
throw new Error("Voice MCP stdio bridge is not configured");
}
return {
...(existing ?? {}),
[VOICE_MCP_SERVER_NAME]: buildVoiceAgentMcpServerConfig({
command: mcpStdio.command,
baseArgs: mcpStdio.baseArgs,
socketPath,
env: mcpStdio.env,
}),
};
}
private getVoiceModeEnforcedModeId(provider: AgentProvider): string | undefined {
return VOICE_MODE_ENFORCED_MODE_ID[provider];
}
private async enableVoiceModeForAgent(agentId: string): Promise<string> {
const ensureVoiceSocket = this.ensureVoiceMcpSocketForAgent;
if (!ensureVoiceSocket) {
throw new Error("Voice MCP socket bridge is not configured");
}
const existing = await this.ensureAgentLoaded(agentId);
const socketPath = await ensureVoiceSocket(agentId);
this.registerVoiceBridgeForAgent(agentId);
const baseConfig: VoiceModeBaseConfig = {
systemPrompt: existing.config.systemPrompt,
mcpServers: this.cloneMcpServers(existing.config.mcpServers),
modeId: existing.config.modeId,
};
this.voiceModeBaseConfig = baseConfig;
const voiceModeId = this.getVoiceModeEnforcedModeId(existing.config.provider);
const refreshOverrides: Partial<AgentSessionConfig> = {
systemPrompt: this.buildVoiceModeSystemPrompt(baseConfig.systemPrompt),
mcpServers: this.buildVoiceModeMcpServers(baseConfig.mcpServers, socketPath),
...(voiceModeId ? { modeId: voiceModeId } : {}),
};
try {
const refreshed = await this.agentManager.refreshAgentFromPersistence(
agentId,
refreshOverrides
);
return refreshed.id;
} catch (error) {
this.unregisterVoiceSpeakHandler?.(agentId);
this.unregisterVoiceCallerContext?.(agentId);
await this.removeVoiceMcpSocketForAgent?.(agentId).catch(() => undefined);
this.voiceModeBaseConfig = null;
throw error;
}
}
private async disableVoiceModeForActiveAgent(
restoreAgentConfig: boolean
): Promise<void> {
const agentId = this.voiceModeAgentId;
if (!agentId) {
this.voiceModeBaseConfig = null;
return;
}
this.unregisterVoiceSpeakHandler?.(agentId);
this.unregisterVoiceCallerContext?.(agentId);
await this.removeVoiceMcpSocketForAgent?.(agentId).catch((error) => {
this.sessionLogger.warn(
{ err: error, agentId },
"Failed to remove voice MCP socket bridge on disable"
);
}); });
return available[0]?.id ?? null;
if (restoreAgentConfig && this.voiceModeBaseConfig) {
const baseConfig = this.voiceModeBaseConfig;
try {
await this.agentManager.refreshAgentFromPersistence(agentId, {
systemPrompt: baseConfig.systemPrompt,
mcpServers: this.cloneMcpServers(baseConfig.mcpServers),
modeId: baseConfig.modeId,
});
} catch (error) {
this.sessionLogger.warn(
{ err: error, agentId },
"Failed to restore agent config while disabling voice mode"
);
}
}
this.voiceModeBaseConfig = null;
this.voiceModeAgentId = null;
} }
/** /**
@@ -4501,101 +4610,11 @@ export class Session {
} }
} }
private async ensureVoiceAssistantAgent(): Promise<string> {
const ensureVoiceSocket = this.ensureVoiceMcpSocketForAgent;
if (!ensureVoiceSocket) {
throw new Error("Voice MCP socket bridge is not configured");
}
if (this.voiceAssistantAgentId) {
const existing = this.agentManager.getAgent(this.voiceAssistantAgentId);
if (existing) {
await ensureVoiceSocket(existing.id);
this.registerVoiceBridgeForAgent(existing.id);
return existing.id;
}
try {
const hydrated = await this.ensureAgentLoaded(this.voiceAssistantAgentId);
await ensureVoiceSocket(hydrated.id);
this.voiceAssistantAgentId = hydrated.id;
this.registerVoiceBridgeForAgent(hydrated.id);
return hydrated.id;
} catch (error) {
this.sessionLogger.debug(
{ err: error, voiceAssistantAgentId: this.voiceAssistantAgentId },
"Voice assistant agent not found in active/persisted state; creating new session"
);
}
}
const provider = this.voiceLlmProvider;
if (!provider) {
throw new Error("Voice LLM provider is not configured");
}
const voiceAgentId = this.voiceAssistantAgentId ?? uuidv4();
const cwd = join(this.paseoHome, "voice-agent-workspace");
await mkdir(cwd, { recursive: true });
const mcpStdio = this.voiceAgentMcpStdio;
if (!mcpStdio) {
throw new Error("Voice MCP stdio bridge is not configured");
}
const socketPath = await ensureVoiceSocket(voiceAgentId);
const model = this.voiceLlmModel?.trim() || undefined;
const config: AgentSessionConfig = {
provider,
cwd,
systemPrompt: VOICE_AGENT_SYSTEM_INSTRUCTION,
modeId: this.voiceLlmModeId ?? "default",
...(model ? { model } : {}),
mcpServers: {
paseo: buildVoiceAgentMcpServerConfig({
command: mcpStdio.command,
baseArgs: mcpStdio.baseArgs,
socketPath,
env: mcpStdio.env,
}),
},
};
const created = await this.agentManager.createAgent(config, voiceAgentId, {
labels: {
surface: "voice",
ui: "true",
voiceClientId: this.clientId,
},
});
this.voiceAssistantAgentId = created.id;
try {
this.registerVoiceBridgeForAgent(created.id);
} catch (error) {
this.unregisterVoiceSpeakHandler?.(created.id);
this.unregisterVoiceCallerContext?.(created.id);
await this.removeVoiceMcpSocketForAgent?.(created.id).catch(() => undefined);
await this.agentManager.closeAgent(created.id).catch(() => undefined);
this.voiceAssistantAgentId = null;
throw error;
}
this.sessionLogger.info(
{
voiceAssistantAgentId: created.id,
provider,
model: model ?? null,
providerExplicit: this.voiceLlmProviderExplicit,
},
"Voice assistant agent initialized"
);
return created.id;
}
private registerVoiceBridgeForAgent(agentId: string): void { private registerVoiceBridgeForAgent(agentId: string): void {
this.registerVoiceSpeakHandler?.(agentId, async ({ text, signal }) => { this.registerVoiceSpeakHandler?.(agentId, async ({ text, signal }) => {
this.sessionLogger.info( this.sessionLogger.info(
{ {
voiceAssistantAgentId: agentId, agentId,
textLength: text.length, textLength: text.length,
preview: text.slice(0, 160), preview: text.slice(0, 160),
}, },
@@ -4609,7 +4628,7 @@ export class Session {
true true
); );
this.sessionLogger.info( this.sessionLogger.info(
{ voiceAssistantAgentId: agentId, textLength: text.length }, { agentId, textLength: text.length },
"Voice speak tool call finished playback" "Voice speak tool call finished playback"
); );
this.emit({ this.emit({
@@ -4625,7 +4644,7 @@ export class Session {
this.registerVoiceCallerContext?.(agentId, { this.registerVoiceCallerContext?.(agentId, {
childAgentDefaultLabels: { ui: "true" }, childAgentDefaultLabels: { ui: "true" },
allowCustomCwd: true, allowCustomCwd: false,
enableVoiceTools: true, enableVoiceTools: true,
}); });
} }
@@ -4643,14 +4662,16 @@ export class Session {
return isVoicePermissionAllowed(request); return isVoicePermissionAllowed(request);
} }
private async processWithVoiceAgent(userText: string): Promise<void> { private async processWithVoiceAgent(
const agentId = await this.ensureVoiceAssistantAgent(); agentId: string,
userText: string
): Promise<void> {
await this.ensureAgentLoaded(agentId);
await this.interruptAgentIfRunning(agentId); await this.interruptAgentIfRunning(agentId);
this.sessionLogger.info( this.sessionLogger.info(
{ {
voiceAssistantAgentId: agentId, agentId,
userTextLength: userText.length, userTextLength: userText.length,
userTextPreview: userText.slice(0, 160), userTextPreview: userText.slice(0, 160),
}, },
@@ -4670,7 +4691,7 @@ export class Session {
if (event.item.type === "tool_call" && typeof event.item.name === "string") { if (event.item.type === "tool_call" && typeof event.item.name === "string") {
this.sessionLogger.info( this.sessionLogger.info(
{ {
voiceAssistantAgentId: agentId, agentId,
toolName: event.item.name, toolName: event.item.name,
callId: event.item.callId, callId: event.item.callId,
}, },
@@ -4689,11 +4710,12 @@ export class Session {
} else { } else {
await this.agentManager.respondToPermission(agentId, event.request.id, { await this.agentManager.respondToPermission(agentId, event.request.id, {
behavior: "deny", behavior: "deny",
message: "Voice assistant policy only allows MCP paseo tools.", message: "Voice assistant policy only allows the speak tool.",
interrupt: true, interrupt: true,
}); });
throw new Error( this.sessionLogger.warn(
`Voice assistant denied non-MCP tool request: ${event.request.name}` { agentId, requestName: event.request.name, requestId: event.request.id },
"Voice assistant denied non-speak permission request"
); );
} }
} }
@@ -4701,12 +4723,12 @@ export class Session {
if (!sawSpeakToolCall) { if (!sawSpeakToolCall) {
this.sessionLogger.warn( this.sessionLogger.warn(
{ voiceAssistantAgentId: agentId }, { agentId },
"Voice agent turn completed without speak tool call; no audio playback emitted" "Voice agent turn completed without speak tool call; no audio playback emitted"
); );
} else { } else {
this.sessionLogger.info( this.sessionLogger.info(
{ voiceAssistantAgentId: agentId }, { agentId },
"Voice agent turn included speak tool call" "Voice agent turn included speak tool call"
); );
} }
@@ -4721,11 +4743,18 @@ export class Session {
this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled"); this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled");
return; return;
} }
const agentId = this.voiceModeAgentId;
if (!agentId) {
this.sessionLogger.warn(
"Ignoring processVoiceTurn call because no agent is currently voice-enabled"
);
return;
}
const normalized = (latestUserText ?? "").trim(); const normalized = (latestUserText ?? "").trim();
if (!normalized) { if (!normalized) {
return; return;
} }
await this.processWithVoiceAgent(normalized); await this.processWithVoiceAgent(agentId, normalized);
} finally { } finally {
this.turnIndex++; this.turnIndex++;
this.currentStreamPromise = null; this.currentStreamPromise = null;
@@ -4980,21 +5009,8 @@ export class Session {
this.agentTools = null; this.agentTools = null;
} }
if (this.voiceAssistantAgentId) { await this.disableVoiceModeForActiveAgent(true);
try { this.isVoiceMode = false;
await this.agentManager.closeAgent(this.voiceAssistantAgentId);
} catch (error) {
this.sessionLogger.warn(
{ err: error, voiceAssistantAgentId: this.voiceAssistantAgentId },
"Failed to close voice assistant agent"
);
} finally {
this.unregisterVoiceSpeakHandler?.(this.voiceAssistantAgentId);
this.unregisterVoiceCallerContext?.(this.voiceAssistantAgentId);
await this.removeVoiceMcpSocketForAgent?.(this.voiceAssistantAgentId).catch(() => undefined);
this.voiceAssistantAgentId = null;
}
}
// Unsubscribe from all terminals // Unsubscribe from all terminals
for (const unsubscribe of this.terminalSubscriptions.values()) { for (const unsubscribe of this.terminalSubscriptions.values()) {
@@ -1,8 +1,7 @@
import { describe, expect, test } from "vitest"; import { describe, expect, test } from "vitest";
import { mkdtempSync } from "node:fs"; import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import { randomUUID } from "node:crypto";
import pino from "pino"; import pino from "pino";
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
@@ -191,7 +190,17 @@ describe("speech models (download E2E)", () => {
}; };
}); });
await ctx.client.setVoiceMode(true, randomUUID()); const voiceCwd = mkdtempSync(path.join(tmpdir(), "speech-download-voice-agent-"));
const voiceAgent = await ctx.client.createAgent({
config: {
provider: "codex",
cwd: voiceCwd,
modeId: "full-access",
model: "gpt-5.1-codex-mini",
thinkingOptionId: "low",
},
});
await ctx.client.setVoiceMode(true, voiceAgent.id);
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) { for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
const isLast = offset + chunkBytes >= pcm16.length; const isLast = offset + chunkBytes >= pcm16.length;
@@ -202,6 +211,7 @@ describe("speech models (download E2E)", () => {
expect(voiceText).toContain("voice note"); expect(voiceText).toContain("voice note");
} }
await ctx.client.setVoiceMode(false); await ctx.client.setVoiceMode(false);
rmSync(voiceCwd, { recursive: true, force: true });
// Streaming TTS: generate locally from downloaded model and validate chunking. // Streaming TTS: generate locally from downloaded model and validate chunking.
const ttsText = "This is a voice note."; const ttsText = "This is a voice note.";
@@ -1,9 +1,12 @@
import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { randomUUID } from "node:crypto";
import path from "node:path"; import path from "node:path";
import { readFile } from "node:fs/promises"; import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import pino from "pino";
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js"; import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
import { getFullAccessConfig } from "./daemon-e2e/agent-configs.js";
import { OpenAITTS } from "./speech/providers/openai/tts.js";
const openaiApiKey = process.env.OPENAI_API_KEY ?? null; const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
const shouldRun = const shouldRun =
@@ -69,61 +72,94 @@ function waitForSignal<T>(
test( test(
"routes voice turns through local agent speak tool", "routes voice turns through local agent speak tool",
async () => { async () => {
await ctx.client.setVoiceMode(true, randomUUID()); const logger = pino({ level: "silent" });
const ttsProvider = new OpenAITTS(
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => { {
const offAudio = ctx.client.on("audio_output", (msg) => { apiKey: openaiApiKey!,
if (msg.type !== "audio_output") return; responseFormat: "pcm",
resolve({ chunkId: msg.payload.id }); voice: "alloy",
}); },
const offError = ctx.client.on("activity_log", (msg) => { logger
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offAudio();
offError();
};
});
const assistantLogPromise = waitForSignal<string>(120000, (resolve, reject) => {
const offLog = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "assistant") return;
const content = String(msg.payload.content ?? "");
if (!content.trim()) return;
resolve(content);
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offLog();
offError();
};
});
const fixturePath = path.resolve(
process.cwd(),
"..",
"app",
"e2e",
"fixtures",
"recording.wav"
); );
const wav = await readFile(fixturePath);
await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true);
const [{ chunkId }, assistantText] = await Promise.all([ const voiceCwd = mkdtempSync(path.join(tmpdir(), "voice-local-agent-"));
audioPromise, const targetAgent = await ctx.client.createAgent({
assistantLogPromise, config: {
...getFullAccessConfig("codex"),
cwd: voiceCwd,
},
});
const voiceMode = await ctx.client.setVoiceMode(true, targetAgent.id);
expect(voiceMode.accepted).toBe(true);
const transcriptionPromise = waitForSignal<string>(120000, (resolve, reject) => {
const offTranscript = ctx.client.on("transcription_result", (msg) => {
if (msg.type !== "transcription_result") return;
const text = String(msg.payload.text ?? "").trim();
if (!text) return;
resolve(text);
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offTranscript();
offError();
};
});
const speakToolPromise = waitForSignal<string>(120000, (resolve, reject) => {
const offStream = ctx.client.on("agent_stream", (msg) => {
if (msg.type !== "agent_stream") return;
if (msg.payload.event.type !== "timeline") return;
const item = msg.payload.event.item;
if (item.type !== "tool_call") return;
const name = String(item.name ?? "");
if (!name.toLowerCase().includes("speak")) return;
resolve(name);
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offStream();
offError();
};
});
const inputSpeech = await ttsProvider.synthesizeSpeech(
"Use the speak tool and say exactly local agent check."
);
const buffers: Buffer[] = [];
for await (const chunk of inputSpeech.stream as AsyncIterable<unknown>) {
buffers.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as ArrayBufferLike));
}
const pcm = Buffer.concat(buffers);
const chunkBytes = 4800;
for (let offset = 0; offset < pcm.length; offset += chunkBytes) {
const chunk = pcm.subarray(offset, Math.min(pcm.length, offset + chunkBytes));
const isLast = offset + chunkBytes >= pcm.length;
await ctx.client.sendVoiceAudioChunk(
chunk.toString("base64"),
"audio/pcm;rate=24000;bits=16",
isLast
);
}
const [transcript, speakToolName] = await Promise.all([
transcriptionPromise,
speakToolPromise,
]); ]);
expect(chunkId.length).toBeGreaterThan(0); await ctx.client.setVoiceMode(false).catch(() => undefined);
expect(assistantText.trim().length).toBeGreaterThan(0); rmSync(voiceCwd, { recursive: true, force: true });
expect(transcript.length).toBeGreaterThan(0);
expect(speakToolName.toLowerCase()).toContain("speak");
const agents = await ctx.client.fetchAgents(); const agents = await ctx.client.fetchAgents();
expect( expect(
@@ -21,13 +21,13 @@ describe("isVoicePermissionAllowed", () => {
expect(result).toBe(true); expect(result).toBe(true);
}); });
test("allows explicit MCP/paseo tool names", () => { test("denies explicit MCP/paseo tool names", () => {
expect( expect(
isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" })) isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))
).toBe(true); ).toBe(false);
expect( expect(
isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" })) isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))
).toBe(true); ).toBe(false);
}); });
test("denies non-tool permission kinds", () => { test("denies non-tool permission kinds", () => {
@@ -37,12 +37,12 @@ describe("isVoicePermissionAllowed", () => {
expect(result).toBe(false); expect(result).toBe(false);
}); });
test("allows codextool only when metadata references MCP/paseo", () => { test("allows codextool only when metadata references speak", () => {
const allowed = isVoicePermissionAllowed( const allowed = isVoicePermissionAllowed(
buildRequest({ buildRequest({
name: "codextool", name: "codextool",
metadata: { metadata: {
questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }], questions: [{ question: "Allow codextool to call speak for user feedback?" }],
}, },
}) })
); );
@@ -1,7 +1,9 @@
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js"; import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"]; const ALLOWED_TOKEN_SET = ["speak"];
const DENIED_TOKEN_SET = [ const DENIED_TOKEN_SET = [
"mcp",
"paseo",
"bash", "bash",
"shell", "shell",
"terminal", "terminal",
@@ -27,10 +29,7 @@ function stringifyMetadata(metadata: unknown): string {
} }
} }
/** /** Voice assistant policy: only allow the speak tool. */
* Voice assistant policy: allow only MCP/paseo/speak tool requests.
* All non-tool permission requests are denied.
*/
export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean { export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean {
if (request.kind !== "tool") { if (request.kind !== "tool") {
return false; return false;
@@ -44,9 +43,6 @@ export function isVoicePermissionAllowed(request: AgentPermissionRequest): boole
if (normalizedName === "speak") { if (normalizedName === "speak") {
return true; return true;
} }
if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) {
return true;
}
if (normalizedName !== "codextool") { if (normalizedName !== "codextool") {
return false; return false;
@@ -1,12 +1,15 @@
import { describe, test, expect, beforeAll, afterAll } from "vitest"; import { describe, test, expect, beforeAll, afterAll } from "vitest";
import { randomUUID } from "node:crypto";
import { Readable } from "node:stream"; import { Readable } from "node:stream";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import pino from "pino"; import pino from "pino";
import { import {
createDaemonTestContext, createDaemonTestContext,
type DaemonTestContext, type DaemonTestContext,
} from "./test-utils/index.js"; } from "./test-utils/index.js";
import { getFullAccessConfig, type AgentProvider } from "./daemon-e2e/agent-configs.js";
import { OpenAITTS } from "./speech/providers/openai/tts.js"; import { OpenAITTS } from "./speech/providers/openai/tts.js";
import { OpenAISTT } from "./speech/providers/openai/stt.js"; import { OpenAISTT } from "./speech/providers/openai/stt.js";
import { STTManager } from "./agent/stt-manager.js"; import { STTManager } from "./agent/stt-manager.js";
@@ -88,8 +91,6 @@ describe("voice roundtrip e2e", () => {
voiceTts: { provider: "openai", explicit: true }, voiceTts: { provider: "openai", explicit: true },
}, },
}, },
voiceLlmProvider: "claude",
voiceLlmProviderExplicit: true,
}); });
}, 60000); }, 60000);
@@ -97,9 +98,10 @@ describe("voice roundtrip e2e", () => {
await ctx.cleanup(); await ctx.cleanup();
}, 60000); }, 60000);
speechTest( for (const targetProvider of ["claude", "codex"] as const satisfies AgentProvider[]) {
"full roundtrip: voice input audio -> voice agent -> output audio -> transcribed output", speechTest(
async () => { `full roundtrip (${targetProvider}): voice input audio -> voice agent -> output audio -> transcribed output`,
async () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const ttsProvider = new OpenAITTS( const ttsProvider = new OpenAITTS(
{ {
@@ -118,7 +120,20 @@ describe("voice roundtrip e2e", () => {
); );
const sttOutput = new STTManager("voice-roundtrip-e2e", logger, sttProvider); const sttOutput = new STTManager("voice-roundtrip-e2e", logger, sttProvider);
const voiceAgentId = randomUUID(); const voiceCwd = mkdtempSync(
path.join(tmpdir(), `voice-roundtrip-agent-${targetProvider}-`)
);
const voiceAgent = await withTimeout(
"createVoiceTargetAgent",
30000,
ctx.client.createAgent({
config: {
...getFullAccessConfig(targetProvider),
cwd: voiceCwd,
},
})
);
const voiceAgentId = voiceAgent.id;
const voiceMode = await withTimeout( const voiceMode = await withTimeout(
"setVoiceMode", "setVoiceMode",
15000, 15000,
@@ -260,6 +275,8 @@ describe("voice roundtrip e2e", () => {
} finally { } finally {
offStream(); offStream();
offErrors(); offErrors();
await ctx.client.setVoiceMode(false).catch(() => undefined);
rmSync(voiceCwd, { recursive: true, force: true });
} }
})(); })();
@@ -281,7 +298,8 @@ describe("voice roundtrip e2e", () => {
expect(normalized.length).toBeGreaterThan(0); expect(normalized.length).toBeGreaterThan(0);
expect(normalized).toMatch(/round|trip|successful/); expect(normalized).toMatch(/round|trip|successful/);
}, },
180000 180000
); );
}
}); });
@@ -83,10 +83,6 @@ export class VoiceAssistantWebSocketServer {
}; };
} | null; } | null;
private readonly voice: { private readonly voice: {
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>; ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>; removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
@@ -110,10 +106,6 @@ export class VoiceAssistantWebSocketServer {
speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null }, speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
terminalManager?: TerminalManager | null, terminalManager?: TerminalManager | null,
voice?: { voice?: {
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>; ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>; removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
+2 -2
View File
@@ -348,7 +348,7 @@ export const ArchiveAgentRequestMessageSchema = z.object({
export const SetVoiceModeMessageSchema = z.object({ export const SetVoiceModeMessageSchema = z.object({
type: z.literal("set_voice_mode"), type: z.literal("set_voice_mode"),
enabled: z.boolean(), enabled: z.boolean(),
voiceAgentId: z.string().optional(), agentId: z.string().optional(),
requestId: z.string().optional(), requestId: z.string().optional(),
}); });
@@ -578,7 +578,7 @@ export const SetVoiceModeResponseMessageSchema = z.object({
payload: z.object({ payload: z.object({
requestId: z.string(), requestId: z.string(),
enabled: z.boolean(), enabled: z.boolean(),
voiceAgentId: z.string().nullable(), agentId: z.string().nullable(),
accepted: z.boolean(), accepted: z.boolean(),
error: z.string().nullable(), error: z.string().nullable(),
}), }),