Update files
This commit is contained in:
@@ -4,10 +4,11 @@ import {
|
|||||||
Text,
|
Text,
|
||||||
ActivityIndicator,
|
ActivityIndicator,
|
||||||
Platform,
|
Platform,
|
||||||
|
Alert,
|
||||||
} from "react-native";
|
} from "react-native";
|
||||||
import { useState, useEffect, useRef, useCallback, useMemo } from "react";
|
import { useState, useEffect, useRef, useCallback, useMemo } from "react";
|
||||||
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||||
import { ArrowUp, Square, Pencil } from "lucide-react-native";
|
import { ArrowUp, Square, Pencil, AudioLines } from "lucide-react-native";
|
||||||
import Animated, { useAnimatedStyle } from "react-native-reanimated";
|
import Animated, { useAnimatedStyle } from "react-native-reanimated";
|
||||||
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
|
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
|
||||||
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
||||||
@@ -30,6 +31,7 @@ import { useAgentCommandsQuery } from "@/hooks/use-agent-commands-query";
|
|||||||
import { encodeImages } from "@/utils/encode-images";
|
import { encodeImages } from "@/utils/encode-images";
|
||||||
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
|
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
|
||||||
import { focusWithRetries } from "@/utils/web-focus";
|
import { focusWithRetries } from "@/utils/web-focus";
|
||||||
|
import { useVoiceOptional } from "@/contexts/voice-context";
|
||||||
|
|
||||||
type QueuedMessage = {
|
type QueuedMessage = {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -76,6 +78,7 @@ export function AgentInputArea({
|
|||||||
const client = useSessionStore(
|
const client = useSessionStore(
|
||||||
(state) => state.sessions[serverId]?.client ?? null
|
(state) => state.sessions[serverId]?.client ?? null
|
||||||
);
|
);
|
||||||
|
const voice = useVoiceOptional();
|
||||||
const isConnected = client?.isConnected ?? false;
|
const isConnected = client?.isConnected ?? false;
|
||||||
|
|
||||||
const agent = useSessionStore((state) =>
|
const agent = useSessionStore((state) =>
|
||||||
@@ -411,6 +414,29 @@ export function AgentInputArea({
|
|||||||
messageInputRef.current?.focus();
|
messageInputRef.current?.focus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const isVoiceModeForAgent =
|
||||||
|
voice?.isVoiceModeForAgent(serverId, agentId) ?? false;
|
||||||
|
|
||||||
|
const handleToggleRealtimeVoice = useCallback(() => {
|
||||||
|
if (!voice || !isConnected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (voice.isVoiceSwitching) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (voice.isVoiceModeForAgent(serverId, agentId)) {
|
||||||
|
void voice.stopVoice().catch((error) => {
|
||||||
|
console.error("[AgentInputArea] Failed to stop voice mode", error);
|
||||||
|
Alert.alert("Voice failed", "Unable to stop realtime voice mode.");
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
void voice.startVoice(serverId, agentId).catch((error) => {
|
||||||
|
console.error("[AgentInputArea] Failed to start voice mode", error);
|
||||||
|
Alert.alert("Voice failed", "Unable to start realtime voice mode.");
|
||||||
|
});
|
||||||
|
}, [agentId, isConnected, serverId, voice]);
|
||||||
|
|
||||||
function handleEditQueuedMessage(id: string) {
|
function handleEditQueuedMessage(id: string) {
|
||||||
const item = queuedMessages.find((q) => q.id === id);
|
const item = queuedMessages.find((q) => q.id === id);
|
||||||
if (!item) return;
|
if (!item) return;
|
||||||
@@ -494,7 +520,7 @@ export function AgentInputArea({
|
|||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
const rightContent = isAgentRunning ? (
|
const cancelButton = isAgentRunning ? (
|
||||||
<Pressable
|
<Pressable
|
||||||
onPress={handleCancelAgent}
|
onPress={handleCancelAgent}
|
||||||
disabled={!isConnected || isCancellingAgent}
|
disabled={!isConnected || isCancellingAgent}
|
||||||
@@ -515,6 +541,35 @@ export function AgentInputArea({
|
|||||||
</Pressable>
|
</Pressable>
|
||||||
) : null;
|
) : null;
|
||||||
|
|
||||||
|
const rightContent = (
|
||||||
|
<View style={styles.rightControls}>
|
||||||
|
<Pressable
|
||||||
|
onPress={handleToggleRealtimeVoice}
|
||||||
|
disabled={!isConnected || voice?.isVoiceSwitching}
|
||||||
|
accessibilityLabel={
|
||||||
|
isVoiceModeForAgent ? "Disable realtime voice mode" : "Enable realtime voice mode"
|
||||||
|
}
|
||||||
|
accessibilityRole="button"
|
||||||
|
style={[
|
||||||
|
styles.realtimeVoiceButton as any,
|
||||||
|
isVoiceModeForAgent ? (styles.realtimeVoiceButtonActive as any) : undefined,
|
||||||
|
(!isConnected || voice?.isVoiceSwitching
|
||||||
|
? styles.buttonDisabled
|
||||||
|
: undefined) as any,
|
||||||
|
]}
|
||||||
|
>
|
||||||
|
{voice?.isVoiceSwitching ? (
|
||||||
|
<ActivityIndicator size="small" color="white" />
|
||||||
|
) : isVoiceModeForAgent ? (
|
||||||
|
<Square size={16} color="white" fill="white" />
|
||||||
|
) : (
|
||||||
|
<AudioLines size={18} color={theme.colors.foreground} />
|
||||||
|
)}
|
||||||
|
</Pressable>
|
||||||
|
{cancelButton}
|
||||||
|
</View>
|
||||||
|
);
|
||||||
|
|
||||||
const leftContent = <AgentStatusBar agentId={agentId} serverId={serverId} />;
|
const leftContent = <AgentStatusBar agentId={agentId} serverId={serverId} />;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -588,6 +643,8 @@ export function AgentInputArea({
|
|||||||
isScreenFocused={isScreenFocused}
|
isScreenFocused={isScreenFocused}
|
||||||
leftContent={leftContent}
|
leftContent={leftContent}
|
||||||
rightContent={rightContent}
|
rightContent={rightContent}
|
||||||
|
voiceServerId={serverId}
|
||||||
|
voiceAgentId={agentId}
|
||||||
isAgentRunning={isAgentRunning}
|
isAgentRunning={isAgentRunning}
|
||||||
onQueue={handleQueue}
|
onQueue={handleQueue}
|
||||||
onKeyPress={handleCommandKeyPress}
|
onKeyPress={handleCommandKeyPress}
|
||||||
@@ -631,6 +688,25 @@ const styles = StyleSheet.create(((theme: Theme) => ({
|
|||||||
alignItems: "center",
|
alignItems: "center",
|
||||||
justifyContent: "center",
|
justifyContent: "center",
|
||||||
},
|
},
|
||||||
|
rightControls: {
|
||||||
|
flexDirection: "row",
|
||||||
|
alignItems: "center",
|
||||||
|
gap: theme.spacing[2],
|
||||||
|
},
|
||||||
|
realtimeVoiceButton: {
|
||||||
|
width: 34,
|
||||||
|
height: 34,
|
||||||
|
borderRadius: theme.borderRadius.full,
|
||||||
|
backgroundColor: theme.colors.surface0,
|
||||||
|
borderWidth: theme.borderWidth[1],
|
||||||
|
borderColor: theme.colors.border,
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
},
|
||||||
|
realtimeVoiceButtonActive: {
|
||||||
|
backgroundColor: theme.colors.palette.green[600],
|
||||||
|
borderColor: theme.colors.palette.green[800],
|
||||||
|
},
|
||||||
buttonDisabled: {
|
buttonDisabled: {
|
||||||
opacity: 0.5,
|
opacity: 0.5,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ export function AgentStreamView({
|
|||||||
const flatListRef = useRef<FlatList<StreamItem>>(null);
|
const flatListRef = useRef<FlatList<StreamItem>>(null);
|
||||||
const { theme } = useUnistyles();
|
const { theme } = useUnistyles();
|
||||||
const insets = useSafeAreaInsets();
|
const insets = useSafeAreaInsets();
|
||||||
const { isVoiceMode } = useVoice();
|
const { isVoiceModeForAgent } = useVoice();
|
||||||
const [isNearBottom, setIsNearBottom] = useState(true);
|
const [isNearBottom, setIsNearBottom] = useState(true);
|
||||||
const hasScrolledInitially = useRef(false);
|
const hasScrolledInitially = useRef(false);
|
||||||
const hasAutoScrolledOnce = useRef(false);
|
const hasAutoScrolledOnce = useRef(false);
|
||||||
@@ -114,6 +114,7 @@ export function AgentStreamView({
|
|||||||
|
|
||||||
// Get serverId (fallback to agent's serverId if not provided)
|
// Get serverId (fallback to agent's serverId if not provided)
|
||||||
const resolvedServerId = serverId ?? agent.serverId ?? "";
|
const resolvedServerId = serverId ?? agent.serverId ?? "";
|
||||||
|
const isVoiceMode = isVoiceModeForAgent(resolvedServerId, agentId);
|
||||||
|
|
||||||
const client = useSessionStore(
|
const client = useSessionStore(
|
||||||
(state) => state.sessions[resolvedServerId]?.client ?? null
|
(state) => state.sessions[resolvedServerId]?.client ?? null
|
||||||
|
|||||||
@@ -61,6 +61,8 @@ export interface MessageInputProps {
|
|||||||
leftContent?: React.ReactNode;
|
leftContent?: React.ReactNode;
|
||||||
/** Content to render on the right side after voice button (e.g., realtime button, cancel button) */
|
/** Content to render on the right side after voice button (e.g., realtime button, cancel button) */
|
||||||
rightContent?: React.ReactNode;
|
rightContent?: React.ReactNode;
|
||||||
|
voiceServerId?: string;
|
||||||
|
voiceAgentId?: string;
|
||||||
/** When true and there's sendable content, calls onQueue instead of onSubmit */
|
/** When true and there's sendable content, calls onQueue instead of onSubmit */
|
||||||
isAgentRunning?: boolean;
|
isAgentRunning?: boolean;
|
||||||
/** Callback for queue button when agent is running */
|
/** Callback for queue button when agent is running */
|
||||||
@@ -117,6 +119,8 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
isScreenFocused = true,
|
isScreenFocused = true,
|
||||||
leftContent,
|
leftContent,
|
||||||
rightContent,
|
rightContent,
|
||||||
|
voiceServerId,
|
||||||
|
voiceAgentId,
|
||||||
isAgentRunning = false,
|
isAgentRunning = false,
|
||||||
onQueue,
|
onQueue,
|
||||||
onKeyPress: onKeyPressCallback,
|
onKeyPress: onKeyPressCallback,
|
||||||
@@ -310,15 +314,20 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
const handleVoicePress = useCallback(async () => {
|
const handleVoicePress = useCallback(async () => {
|
||||||
|
const isRealtimeVoiceForCurrentAgent =
|
||||||
|
voice &&
|
||||||
|
!!voiceServerId &&
|
||||||
|
!!voiceAgentId &&
|
||||||
|
voice.isVoiceModeForAgent(voiceServerId, voiceAgentId);
|
||||||
if (isDictating) {
|
if (isDictating) {
|
||||||
await cancelDictation();
|
await cancelDictation();
|
||||||
} else {
|
} else {
|
||||||
if (voice?.isVoiceMode) {
|
if (isRealtimeVoiceForCurrentAgent) {
|
||||||
await voice.stopVoice();
|
await voice.stopVoice();
|
||||||
}
|
}
|
||||||
await startDictation();
|
await startDictation();
|
||||||
}
|
}
|
||||||
}, [isDictating, cancelDictation, startDictation, voice]);
|
}, [isDictating, cancelDictation, startDictation, voice, voiceAgentId, voiceServerId]);
|
||||||
|
|
||||||
const handleCancelRecording = useCallback(async () => {
|
const handleCancelRecording = useCallback(async () => {
|
||||||
await cancelDictation();
|
await cancelDictation();
|
||||||
@@ -626,6 +635,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
<Mic size={20} color={theme.colors.foreground} />
|
<Mic size={20} color={theme.colors.foreground} />
|
||||||
)}
|
)}
|
||||||
</Pressable>
|
</Pressable>
|
||||||
|
{rightContent}
|
||||||
{shouldShowSendButton && (
|
{shouldShowSendButton && (
|
||||||
<Pressable
|
<Pressable
|
||||||
onPress={handleSendMessage}
|
onPress={handleSendMessage}
|
||||||
@@ -647,7 +657,6 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
<ArrowUp size={20} color="white" />
|
<ArrowUp size={20} color="white" />
|
||||||
</Pressable>
|
</Pressable>
|
||||||
)}
|
)}
|
||||||
{!shouldShowSendButton && rightContent}
|
|
||||||
</View>
|
</View>
|
||||||
</View>
|
</View>
|
||||||
</Animated.View>
|
</Animated.View>
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { useCallback, useMemo, useState, useEffect } from "react";
|
import { useCallback, useMemo, useState, useEffect } from "react";
|
||||||
import { View, Pressable, Text, Platform, Alert, ActivityIndicator } from "react-native";
|
import { View, Pressable, Text, Platform } from "react-native";
|
||||||
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
||||||
import Animated, {
|
import Animated, {
|
||||||
useAnimatedStyle,
|
useAnimatedStyle,
|
||||||
@@ -9,21 +9,17 @@ import Animated, {
|
|||||||
} from "react-native-reanimated";
|
} from "react-native-reanimated";
|
||||||
import { Gesture, GestureDetector } from "react-native-gesture-handler";
|
import { Gesture, GestureDetector } from "react-native-gesture-handler";
|
||||||
import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles";
|
import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles";
|
||||||
import { Plus, Settings, Users, AudioLines } from "lucide-react-native";
|
import { Plus, Settings, Users } from "lucide-react-native";
|
||||||
import { router } from "expo-router";
|
import { router } from "expo-router";
|
||||||
import { usePanelStore } from "@/stores/panel-store";
|
import { usePanelStore } from "@/stores/panel-store";
|
||||||
import { GroupedAgentList } from "./grouped-agent-list";
|
import { GroupedAgentList } from "./grouped-agent-list";
|
||||||
import { useAggregatedAgents } from "@/hooks/use-aggregated-agents";
|
import { useAggregatedAgents } from "@/hooks/use-aggregated-agents";
|
||||||
import { useSidebarAnimation } from "@/contexts/sidebar-animation-context";
|
import { useSidebarAnimation } from "@/contexts/sidebar-animation-context";
|
||||||
import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window";
|
import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window";
|
||||||
import { useVoice } from "@/contexts/voice-context";
|
|
||||||
import { useDaemonConnections } from "@/contexts/daemon-connections-context";
|
|
||||||
import { VoicePanel } from "./voice-panel";
|
|
||||||
import { useSidebarAgentSections } from "@/hooks/use-sidebar-agent-sections";
|
import { useSidebarAgentSections } from "@/hooks/use-sidebar-agent-sections";
|
||||||
import { useSidebarCollapsedSectionsStore } from "@/stores/sidebar-collapsed-sections-store";
|
import { useSidebarCollapsedSectionsStore } from "@/stores/sidebar-collapsed-sections-store";
|
||||||
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
|
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
|
||||||
import { deriveSidebarShortcutAgentKeys } from "@/utils/sidebar-shortcuts";
|
import { deriveSidebarShortcutAgentKeys } from "@/utils/sidebar-shortcuts";
|
||||||
import { AdaptiveModalSheet } from "@/components/adaptive-modal-sheet";
|
|
||||||
|
|
||||||
const DESKTOP_SIDEBAR_WIDTH = 320;
|
const DESKTOP_SIDEBAR_WIDTH = 320;
|
||||||
const SIDEBAR_AGENT_LIMIT = 50; // temporary for screenshots
|
const SIDEBAR_AGENT_LIMIT = 50; // temporary for screenshots
|
||||||
@@ -56,9 +52,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
} = useSidebarAnimation();
|
} = useSidebarAnimation();
|
||||||
const trafficLightPadding = useTrafficLightPadding();
|
const trafficLightPadding = useTrafficLightPadding();
|
||||||
const dragHandlers = useTauriDragHandlers();
|
const dragHandlers = useTauriDragHandlers();
|
||||||
const { connectionStates } = useDaemonConnections();
|
|
||||||
const { isVoiceMode, isVoiceSwitching, startVoice, stopVoice } = useVoice();
|
|
||||||
const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false);
|
|
||||||
|
|
||||||
// Track user-initiated refresh to avoid showing spinner on background revalidation
|
// Track user-initiated refresh to avoid showing spinner on background revalidation
|
||||||
const [isManualRefresh, setIsManualRefresh] = useState(false);
|
const [isManualRefresh, setIsManualRefresh] = useState(false);
|
||||||
@@ -142,65 +135,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
router.push("/agents");
|
router.push("/agents");
|
||||||
}, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]);
|
}, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]);
|
||||||
|
|
||||||
const voiceEligibleHosts = useMemo(() => {
|
|
||||||
return Array.from(connectionStates.values()).filter((entry) => entry.status === "online");
|
|
||||||
}, [connectionStates]);
|
|
||||||
const hasAnyConfiguredHosts = connectionStates.size > 0;
|
|
||||||
|
|
||||||
const handleToggleVoice = useCallback(() => {
|
|
||||||
if (isVoiceSwitching) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (isVoiceMode) {
|
|
||||||
void stopVoice().catch((error) => {
|
|
||||||
console.error("[SlidingSidebar] Failed to stop voice", error);
|
|
||||||
Alert.alert("Voice failed", "Unable to stop voice mode.");
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (voiceEligibleHosts.length === 0) {
|
|
||||||
if (!hasAnyConfiguredHosts) {
|
|
||||||
Alert.alert(
|
|
||||||
"No hosts available",
|
|
||||||
"Add a host in Settings before starting Voice mode.",
|
|
||||||
[{ text: "Open Settings", onPress: () => router.push("/settings") }, { text: "OK" }]
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Alert.alert(
|
|
||||||
"Hosts reconnecting",
|
|
||||||
"Every host is offline right now. Paseo reconnects automatically—try Voice mode again once one comes online."
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (voiceEligibleHosts.length === 1) {
|
|
||||||
void startVoice(voiceEligibleHosts[0].daemon.serverId).catch((error) => {
|
|
||||||
console.error("[SlidingSidebar] Failed to start voice", error);
|
|
||||||
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
setShowVoiceHostPicker(true);
|
|
||||||
}, [hasAnyConfiguredHosts, isVoiceMode, isVoiceSwitching, startVoice, stopVoice, voiceEligibleHosts]);
|
|
||||||
|
|
||||||
const handleSelectVoiceHost = useCallback(
|
|
||||||
(serverId: string) => {
|
|
||||||
setShowVoiceHostPicker(false);
|
|
||||||
void startVoice(serverId).catch((error) => {
|
|
||||||
console.error("[SlidingSidebar] Failed to start voice", error);
|
|
||||||
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
|
|
||||||
});
|
|
||||||
},
|
|
||||||
[startVoice]
|
|
||||||
);
|
|
||||||
|
|
||||||
const handleDismissVoiceHostPicker = useCallback(() => {
|
|
||||||
setShowVoiceHostPicker(false);
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// Close gesture (swipe left to close when sidebar is open)
|
// Close gesture (swipe left to close when sidebar is open)
|
||||||
// Only activates on leftward swipe, fails on rightward or vertical movement
|
// Only activates on leftward swipe, fails on rightward or vertical movement
|
||||||
// This mirrors the explorer-sidebar pattern for the right sidebar
|
// This mirrors the explorer-sidebar pattern for the right sidebar
|
||||||
@@ -301,8 +235,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
parentGestureRef={closeGestureRef}
|
parentGestureRef={closeGestureRef}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
{isVoiceMode ? <VoicePanel /> : null}
|
|
||||||
|
|
||||||
{/* Footer */}
|
{/* Footer */}
|
||||||
<View style={styles.sidebarFooter}>
|
<View style={styles.sidebarFooter}>
|
||||||
<Pressable
|
<Pressable
|
||||||
@@ -319,43 +251,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
)}
|
)}
|
||||||
</Pressable>
|
</Pressable>
|
||||||
<View style={styles.footerIconRow}>
|
<View style={styles.footerIconRow}>
|
||||||
<Pressable
|
|
||||||
style={styles.footerIconButton}
|
|
||||||
testID="sidebar-voice"
|
|
||||||
nativeID="sidebar-voice"
|
|
||||||
collapsable={false}
|
|
||||||
accessible
|
|
||||||
accessibilityLabel="Voice"
|
|
||||||
accessibilityRole="button"
|
|
||||||
disabled={isVoiceSwitching}
|
|
||||||
onPress={handleToggleVoice}
|
|
||||||
>
|
|
||||||
{({ hovered }) => (
|
|
||||||
isVoiceSwitching ? (
|
|
||||||
<ActivityIndicator
|
|
||||||
size="small"
|
|
||||||
color={
|
|
||||||
isVoiceMode
|
|
||||||
? theme.colors.foreground
|
|
||||||
: hovered
|
|
||||||
? theme.colors.foreground
|
|
||||||
: theme.colors.foregroundMuted
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
) : (
|
|
||||||
<AudioLines
|
|
||||||
size={20}
|
|
||||||
color={
|
|
||||||
isVoiceMode
|
|
||||||
? theme.colors.foreground
|
|
||||||
: hovered
|
|
||||||
? theme.colors.foreground
|
|
||||||
: theme.colors.foregroundMuted
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
)
|
|
||||||
)}
|
|
||||||
</Pressable>
|
|
||||||
<Pressable
|
<Pressable
|
||||||
style={styles.footerIconButton}
|
style={styles.footerIconButton}
|
||||||
testID="sidebar-settings"
|
testID="sidebar-settings"
|
||||||
@@ -375,32 +270,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
</View>
|
</View>
|
||||||
</Animated.View>
|
</Animated.View>
|
||||||
</GestureDetector>
|
</GestureDetector>
|
||||||
|
|
||||||
<AdaptiveModalSheet
|
|
||||||
title="Choose a host"
|
|
||||||
visible={showVoiceHostPicker}
|
|
||||||
onClose={handleDismissVoiceHostPicker}
|
|
||||||
snapPoints={["45%", "70%"]}
|
|
||||||
testID="voice-host-picker"
|
|
||||||
>
|
|
||||||
<View style={styles.hostPickerList}>
|
|
||||||
{voiceEligibleHosts.map((entry) => (
|
|
||||||
<Pressable
|
|
||||||
key={entry.daemon.serverId}
|
|
||||||
style={styles.hostPickerOption}
|
|
||||||
onPress={() => handleSelectVoiceHost(entry.daemon.serverId)}
|
|
||||||
>
|
|
||||||
<Text style={styles.hostPickerOptionText}>{entry.daemon.label}</Text>
|
|
||||||
</Pressable>
|
|
||||||
))}
|
|
||||||
<Pressable
|
|
||||||
style={styles.hostPickerCancel}
|
|
||||||
onPress={handleDismissVoiceHostPicker}
|
|
||||||
>
|
|
||||||
<Text style={styles.hostPickerCancelText}>Cancel</Text>
|
|
||||||
</Pressable>
|
|
||||||
</View>
|
|
||||||
</AdaptiveModalSheet>
|
|
||||||
</View>
|
</View>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -439,8 +308,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
selectedAgentId={selectedAgentId}
|
selectedAgentId={selectedAgentId}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
{isVoiceMode ? <VoicePanel /> : null}
|
|
||||||
|
|
||||||
{/* Footer */}
|
{/* Footer */}
|
||||||
<View style={styles.sidebarFooter}>
|
<View style={styles.sidebarFooter}>
|
||||||
<Pressable
|
<Pressable
|
||||||
@@ -457,43 +324,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
)}
|
)}
|
||||||
</Pressable>
|
</Pressable>
|
||||||
<View style={styles.footerIconRow}>
|
<View style={styles.footerIconRow}>
|
||||||
<Pressable
|
|
||||||
style={styles.footerIconButton}
|
|
||||||
testID="sidebar-voice"
|
|
||||||
nativeID="sidebar-voice"
|
|
||||||
collapsable={false}
|
|
||||||
accessible
|
|
||||||
accessibilityLabel="Voice"
|
|
||||||
accessibilityRole="button"
|
|
||||||
disabled={isVoiceSwitching}
|
|
||||||
onPress={handleToggleVoice}
|
|
||||||
>
|
|
||||||
{({ hovered }) => (
|
|
||||||
isVoiceSwitching ? (
|
|
||||||
<ActivityIndicator
|
|
||||||
size="small"
|
|
||||||
color={
|
|
||||||
isVoiceMode
|
|
||||||
? theme.colors.foreground
|
|
||||||
: hovered
|
|
||||||
? theme.colors.foreground
|
|
||||||
: theme.colors.foregroundMuted
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
) : (
|
|
||||||
<AudioLines
|
|
||||||
size={20}
|
|
||||||
color={
|
|
||||||
isVoiceMode
|
|
||||||
? theme.colors.foreground
|
|
||||||
: hovered
|
|
||||||
? theme.colors.foreground
|
|
||||||
: theme.colors.foregroundMuted
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
)
|
|
||||||
)}
|
|
||||||
</Pressable>
|
|
||||||
<Pressable
|
<Pressable
|
||||||
style={styles.footerIconButton}
|
style={styles.footerIconButton}
|
||||||
testID="sidebar-settings"
|
testID="sidebar-settings"
|
||||||
@@ -510,32 +340,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
</Pressable>
|
</Pressable>
|
||||||
</View>
|
</View>
|
||||||
</View>
|
</View>
|
||||||
|
|
||||||
<AdaptiveModalSheet
|
|
||||||
title="Choose a host"
|
|
||||||
visible={showVoiceHostPicker}
|
|
||||||
onClose={handleDismissVoiceHostPicker}
|
|
||||||
snapPoints={["45%", "70%"]}
|
|
||||||
testID="voice-host-picker"
|
|
||||||
>
|
|
||||||
<View style={styles.hostPickerList}>
|
|
||||||
{voiceEligibleHosts.map((entry) => (
|
|
||||||
<Pressable
|
|
||||||
key={entry.daemon.serverId}
|
|
||||||
style={styles.hostPickerOption}
|
|
||||||
onPress={() => handleSelectVoiceHost(entry.daemon.serverId)}
|
|
||||||
>
|
|
||||||
<Text style={styles.hostPickerOptionText}>{entry.daemon.label}</Text>
|
|
||||||
</Pressable>
|
|
||||||
))}
|
|
||||||
<Pressable
|
|
||||||
style={styles.hostPickerCancel}
|
|
||||||
onPress={handleDismissVoiceHostPicker}
|
|
||||||
>
|
|
||||||
<Text style={styles.hostPickerCancelText}>Cancel</Text>
|
|
||||||
</Pressable>
|
|
||||||
</View>
|
|
||||||
</AdaptiveModalSheet>
|
|
||||||
</View>
|
</View>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,10 +18,12 @@ interface VoiceContextValue {
|
|||||||
isDetecting: boolean;
|
isDetecting: boolean;
|
||||||
isSpeaking: boolean;
|
isSpeaking: boolean;
|
||||||
segmentDuration: number;
|
segmentDuration: number;
|
||||||
startVoice: (serverId: string) => Promise<void>;
|
startVoice: (serverId: string, agentId: string) => Promise<void>;
|
||||||
stopVoice: () => Promise<void>;
|
stopVoice: () => Promise<void>;
|
||||||
|
isVoiceModeForAgent: (serverId: string, agentId: string) => boolean;
|
||||||
toggleMute: () => void;
|
toggleMute: () => void;
|
||||||
activeServerId: string | null;
|
activeServerId: string | null;
|
||||||
|
activeAgentId: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const VoiceContext = createContext<VoiceContextValue | null>(null);
|
const VoiceContext = createContext<VoiceContextValue | null>(null);
|
||||||
@@ -45,6 +47,7 @@ interface VoiceProviderProps {
|
|||||||
export function VoiceProvider({ children }: VoiceProviderProps) {
|
export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||||
const getSession = useSessionStore((state) => state.getSession);
|
const getSession = useSessionStore((state) => state.getSession);
|
||||||
const [activeServerId, setActiveServerId] = useState<string | null>(null);
|
const [activeServerId, setActiveServerId] = useState<string | null>(null);
|
||||||
|
const [activeAgentId, setActiveAgentId] = useState<string | null>(null);
|
||||||
const activeSession = useSessionStore(
|
const activeSession = useSessionStore(
|
||||||
useCallback(
|
useCallback(
|
||||||
(state: ReturnType<typeof useSessionStore.getState>) => {
|
(state: ReturnType<typeof useSessionStore.getState>) => {
|
||||||
@@ -151,7 +154,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
voiceTransportReadyRef.current = false;
|
voiceTransportReadyRef.current = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isVoiceMode || !activeServerId || !client) {
|
if (!isVoiceMode || !activeServerId || !activeAgentId || !client) {
|
||||||
wasVoiceSocketConnectedRef.current = connected;
|
wasVoiceSocketConnectedRef.current = connected;
|
||||||
if (!isVoiceMode) {
|
if (!isVoiceMode) {
|
||||||
lastVoiceModeSyncedClientRef.current = null;
|
lastVoiceModeSyncedClientRef.current = null;
|
||||||
@@ -167,7 +170,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
voiceResyncInFlightRef.current = true;
|
voiceResyncInFlightRef.current = true;
|
||||||
voiceTransportReadyRef.current = false;
|
voiceTransportReadyRef.current = false;
|
||||||
setIsVoiceSwitching(true);
|
setIsVoiceSwitching(true);
|
||||||
void client.setVoiceMode(true).then(
|
void client.setVoiceMode(true, activeAgentId).then(
|
||||||
() => {
|
() => {
|
||||||
console.log("[Voice] Re-synced voice mode after reconnect");
|
console.log("[Voice] Re-synced voice mode after reconnect");
|
||||||
lastVoiceModeSyncedClientRef.current = client;
|
lastVoiceModeSyncedClientRef.current = client;
|
||||||
@@ -184,7 +187,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
wasVoiceSocketConnectedRef.current = connected;
|
wasVoiceSocketConnectedRef.current = connected;
|
||||||
}, [activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]);
|
}, [activeAgentId, activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]);
|
||||||
|
|
||||||
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
|
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
|
||||||
|
|
||||||
@@ -201,7 +204,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
}, [isPlayingAudio]);
|
}, [isPlayingAudio]);
|
||||||
|
|
||||||
const startVoice = useCallback(
|
const startVoice = useCallback(
|
||||||
async (serverId: string) => {
|
async (serverId: string, agentId: string) => {
|
||||||
const session = getSession(serverId) ?? null;
|
const session = getSession(serverId) ?? null;
|
||||||
if (!session) {
|
if (!session) {
|
||||||
throw new Error(`Host ${serverId} is not connected`);
|
throw new Error(`Host ${serverId} is not connected`);
|
||||||
@@ -210,13 +213,23 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
setIsVoiceSwitching(true);
|
setIsVoiceSwitching(true);
|
||||||
voiceTransportReadyRef.current = false;
|
voiceTransportReadyRef.current = false;
|
||||||
try {
|
try {
|
||||||
|
const previousSession = realtimeSessionRef.current;
|
||||||
|
if (
|
||||||
|
isVoiceMode &&
|
||||||
|
previousSession?.client &&
|
||||||
|
(activeServerId !== serverId || activeAgentId !== agentId)
|
||||||
|
) {
|
||||||
|
await previousSession.client.setVoiceMode(false);
|
||||||
|
}
|
||||||
|
|
||||||
realtimeSessionRef.current = session;
|
realtimeSessionRef.current = session;
|
||||||
setActiveServerId(serverId);
|
setActiveServerId(serverId);
|
||||||
|
setActiveAgentId(agentId);
|
||||||
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
|
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
|
||||||
console.warn("[Voice] Failed to activate keep-awake:", error);
|
console.warn("[Voice] Failed to activate keep-awake:", error);
|
||||||
});
|
});
|
||||||
if (session?.client) {
|
if (session?.client) {
|
||||||
await session.client.setVoiceMode(true);
|
await session.client.setVoiceMode(true, agentId);
|
||||||
} else {
|
} else {
|
||||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||||
}
|
}
|
||||||
@@ -230,13 +243,14 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
console.error("[Voice] Failed to start:", error);
|
console.error("[Voice] Failed to start:", error);
|
||||||
await realtimeAudio.stop().catch(() => undefined);
|
await realtimeAudio.stop().catch(() => undefined);
|
||||||
setActiveServerId((current) => (current === serverId ? null : current));
|
setActiveServerId((current) => (current === serverId ? null : current));
|
||||||
|
setActiveAgentId((current) => (current === agentId ? null : current));
|
||||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||||
throw error;
|
throw error;
|
||||||
} finally {
|
} finally {
|
||||||
setIsVoiceSwitching(false);
|
setIsVoiceSwitching(false);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[getSession, realtimeAudio]
|
[activeAgentId, activeServerId, getSession, isVoiceMode, realtimeAudio]
|
||||||
);
|
);
|
||||||
|
|
||||||
const stopVoice = useCallback(async () => {
|
const stopVoice = useCallback(async () => {
|
||||||
@@ -254,6 +268,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
await realtimeAudio.stop();
|
await realtimeAudio.stop();
|
||||||
setIsVoiceMode(false);
|
setIsVoiceMode(false);
|
||||||
setActiveServerId(null);
|
setActiveServerId(null);
|
||||||
|
setActiveAgentId(null);
|
||||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||||
console.log("[Voice] Mode disabled");
|
console.log("[Voice] Mode disabled");
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
@@ -265,6 +280,12 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
}
|
}
|
||||||
}, [realtimeAudio]);
|
}, [realtimeAudio]);
|
||||||
|
|
||||||
|
const isVoiceModeForAgent = useCallback(
|
||||||
|
(serverId: string, agentId: string) =>
|
||||||
|
isVoiceMode && activeServerId === serverId && activeAgentId === agentId,
|
||||||
|
[activeAgentId, activeServerId, isVoiceMode]
|
||||||
|
);
|
||||||
|
|
||||||
const value: VoiceContextValue = {
|
const value: VoiceContextValue = {
|
||||||
isVoiceMode,
|
isVoiceMode,
|
||||||
isVoiceSwitching,
|
isVoiceSwitching,
|
||||||
@@ -275,8 +296,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
segmentDuration: realtimeAudio.segmentDuration,
|
segmentDuration: realtimeAudio.segmentDuration,
|
||||||
startVoice,
|
startVoice,
|
||||||
stopVoice,
|
stopVoice,
|
||||||
|
isVoiceModeForAgent,
|
||||||
toggleMute: realtimeAudio.toggleMute,
|
toggleMute: realtimeAudio.toggleMute,
|
||||||
activeServerId,
|
activeServerId,
|
||||||
|
activeAgentId,
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
import { randomUUID } from "node:crypto";
|
import { mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
import pino from "pino";
|
import pino from "pino";
|
||||||
|
|
||||||
import { createTestPaseoDaemon } from "../src/server/test-utils/paseo-daemon.js";
|
import { createTestPaseoDaemon } from "../src/server/test-utils/paseo-daemon.js";
|
||||||
@@ -48,9 +50,20 @@ async function main(): Promise<void> {
|
|||||||
await client.connect();
|
await client.connect();
|
||||||
client.subscribeAgentUpdates({ subscriptionId: "voice-debug" });
|
client.subscribeAgentUpdates({ subscriptionId: "voice-debug" });
|
||||||
|
|
||||||
const voiceAgentId = randomUUID();
|
const voiceCwd = mkdtempSync(path.join(tmpdir(), "voice-roundtrip-debug-"));
|
||||||
|
const voiceAgent = await client.createAgent({
|
||||||
|
config: {
|
||||||
|
provider: "claude",
|
||||||
|
cwd: voiceCwd,
|
||||||
|
modeId: "bypassPermissions",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const voiceAgentId = voiceAgent.id;
|
||||||
const mode = await client.setVoiceMode(true, voiceAgentId);
|
const mode = await client.setVoiceMode(true, voiceAgentId);
|
||||||
console.log("set_voice_mode_response", mode);
|
console.log("set_voice_mode_response", mode);
|
||||||
|
if (!mode.accepted) {
|
||||||
|
throw new Error(`setVoiceMode rejected: ${mode.error ?? "unknown error"}`);
|
||||||
|
}
|
||||||
|
|
||||||
const offTranscript = client.on("transcription_result", (msg) => {
|
const offTranscript = client.on("transcription_result", (msg) => {
|
||||||
if (msg.type !== "transcription_result") return;
|
if (msg.type !== "transcription_result") return;
|
||||||
@@ -149,6 +162,7 @@ async function main(): Promise<void> {
|
|||||||
|
|
||||||
console.log("success", { audioChunkCount });
|
console.log("success", { audioChunkCount });
|
||||||
await client.setVoiceMode(false);
|
await client.setVoiceMode(false);
|
||||||
|
rmSync(voiceCwd, { recursive: true, force: true });
|
||||||
|
|
||||||
offTranscript();
|
offTranscript();
|
||||||
offActivity();
|
offActivity();
|
||||||
|
|||||||
@@ -1291,13 +1291,13 @@ export class DaemonClient {
|
|||||||
|
|
||||||
async setVoiceMode(
|
async setVoiceMode(
|
||||||
enabled: boolean,
|
enabled: boolean,
|
||||||
voiceAgentId?: string
|
agentId?: string
|
||||||
): Promise<SetVoiceModePayload> {
|
): Promise<SetVoiceModePayload> {
|
||||||
const requestId = this.createRequestId();
|
const requestId = this.createRequestId();
|
||||||
const message = SessionInboundMessageSchema.parse({
|
const message = SessionInboundMessageSchema.parse({
|
||||||
type: "set_voice_mode",
|
type: "set_voice_mode",
|
||||||
enabled,
|
enabled,
|
||||||
...(voiceAgentId ? { voiceAgentId } : {}),
|
...(agentId ? { agentId } : {}),
|
||||||
requestId,
|
requestId,
|
||||||
});
|
});
|
||||||
return this.sendRequest({
|
return this.sendRequest({
|
||||||
|
|||||||
@@ -407,22 +407,24 @@ export class AgentManager {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async refreshAgentFromPersistence(agentId: string): Promise<ManagedAgent> {
|
async refreshAgentFromPersistence(
|
||||||
|
agentId: string,
|
||||||
|
overrides?: Partial<AgentSessionConfig>
|
||||||
|
): Promise<ManagedAgent> {
|
||||||
const existing = this.requireAgent(agentId);
|
const existing = this.requireAgent(agentId);
|
||||||
const handle = existing.persistence;
|
const handle = existing.persistence;
|
||||||
if (!handle) {
|
const provider = handle?.provider ?? existing.provider;
|
||||||
throw new Error(
|
const client = this.requireClient(provider);
|
||||||
`Agent ${agentId} cannot be refreshed because it has no persistence handle`
|
const refreshConfig = {
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const client = this.requireClient(handle.provider);
|
|
||||||
const overrides = {
|
|
||||||
...existing.config,
|
...existing.config,
|
||||||
provider: handle.provider,
|
...overrides,
|
||||||
};
|
provider,
|
||||||
|
} as AgentSessionConfig;
|
||||||
|
const normalizedConfig = await this.normalizeConfig(refreshConfig);
|
||||||
|
|
||||||
const session = await client.resumeSession(handle, overrides);
|
const session = handle
|
||||||
|
? await client.resumeSession(handle, normalizedConfig)
|
||||||
|
: await client.createSession(normalizedConfig);
|
||||||
|
|
||||||
// Remove the existing agent entry before swapping sessions
|
// Remove the existing agent entry before swapping sessions
|
||||||
this.agents.delete(agentId);
|
this.agents.delete(agentId);
|
||||||
@@ -436,7 +438,12 @@ export class AgentManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Preserve existing labels during refresh
|
// Preserve existing labels during refresh
|
||||||
return this.registerSession(session, overrides, agentId, { labels: existing.labels });
|
return this.registerSession(session, normalizedConfig, agentId, {
|
||||||
|
labels: existing.labels,
|
||||||
|
createdAt: existing.createdAt,
|
||||||
|
updatedAt: existing.updatedAt,
|
||||||
|
lastUserMessageAt: existing.lastUserMessageAt,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async closeAgent(agentId: string): Promise<void> {
|
async closeAgent(agentId: string): Promise<void> {
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ export interface AgentMcpServerOptions {
|
|||||||
callerAgentId: string
|
callerAgentId: string
|
||||||
) => VoiceCallerContext | null;
|
) => VoiceCallerContext | null;
|
||||||
enableVoiceTools?: boolean;
|
enableVoiceTools?: boolean;
|
||||||
|
voiceOnly?: boolean;
|
||||||
logger: Logger;
|
logger: Logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -393,7 +394,7 @@ export async function createAgentMcpServer(
|
|||||||
initialMode: topLevelInputSchema.initialMode.optional(),
|
initialMode: topLevelInputSchema.initialMode.optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (options.enableVoiceTools || callerContext?.enableVoiceTools) {
|
if (options.voiceOnly || options.enableVoiceTools || callerContext?.enableVoiceTools) {
|
||||||
server.registerTool(
|
server.registerTool(
|
||||||
"speak",
|
"speak",
|
||||||
{
|
{
|
||||||
@@ -432,6 +433,10 @@ export async function createAgentMcpServer(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (options.voiceOnly) {
|
||||||
|
return server;
|
||||||
|
}
|
||||||
|
|
||||||
server.registerTool(
|
server.registerTool(
|
||||||
"create_agent",
|
"create_agent",
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -64,7 +64,6 @@ import type {
|
|||||||
AgentClient,
|
AgentClient,
|
||||||
AgentProvider,
|
AgentProvider,
|
||||||
} from "./agent/agent-sdk-types.js";
|
} from "./agent/agent-sdk-types.js";
|
||||||
import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js";
|
|
||||||
import { acquirePidLock, releasePidLock } from "./pid-lock.js";
|
import { acquirePidLock, releasePidLock } from "./pid-lock.js";
|
||||||
import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js";
|
import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js";
|
||||||
import {
|
import {
|
||||||
@@ -278,98 +277,8 @@ export async function createPaseoDaemon(
|
|||||||
logger.info(
|
logger.info(
|
||||||
`Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`
|
`Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`
|
||||||
);
|
);
|
||||||
|
|
||||||
const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null;
|
|
||||||
const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false;
|
|
||||||
const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS
|
|
||||||
.filter((definition) => definition.voice?.enabled)
|
|
||||||
.map((definition) => definition.id as AgentProvider);
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
"Voice mode configured for agent-scoped resume flow (no dedicated voice assistant provider)"
|
||||||
requestedVoiceLlmProvider,
|
|
||||||
voiceLlmProviderExplicit,
|
|
||||||
voiceEnabledProviders,
|
|
||||||
},
|
|
||||||
"Voice LLM provider reconciliation started"
|
|
||||||
);
|
|
||||||
|
|
||||||
const providerClients = createAllClients(logger);
|
|
||||||
Object.assign(providerClients, config.agentClients);
|
|
||||||
const voiceLlmAvailability = Object.fromEntries(
|
|
||||||
voiceEnabledProviders.map((provider) => [provider, false])
|
|
||||||
) as Record<AgentProvider, boolean>;
|
|
||||||
for (const provider of voiceEnabledProviders) {
|
|
||||||
try {
|
|
||||||
voiceLlmAvailability[provider] = await providerClients[provider].isAvailable();
|
|
||||||
} catch (error) {
|
|
||||||
logger.warn({ err: error, provider }, "Voice LLM provider availability check failed");
|
|
||||||
voiceLlmAvailability[provider] = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let resolvedVoiceLlmProvider: AgentProvider | null = null;
|
|
||||||
if (requestedVoiceLlmProvider) {
|
|
||||||
if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) {
|
|
||||||
logger.error(
|
|
||||||
{ provider: requestedVoiceLlmProvider, voiceEnabledProviders },
|
|
||||||
"Configured voice LLM provider does not support voice mode"
|
|
||||||
);
|
|
||||||
throw new Error(
|
|
||||||
`Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (!voiceLlmAvailability[requestedVoiceLlmProvider]) {
|
|
||||||
logger.error(
|
|
||||||
{ provider: requestedVoiceLlmProvider, voiceLlmAvailability },
|
|
||||||
"Configured voice LLM provider is unavailable"
|
|
||||||
);
|
|
||||||
throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`);
|
|
||||||
}
|
|
||||||
resolvedVoiceLlmProvider = requestedVoiceLlmProvider;
|
|
||||||
} else {
|
|
||||||
resolvedVoiceLlmProvider =
|
|
||||||
voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
let resolvedVoiceLlmModeId: string | null = null;
|
|
||||||
let resolvedVoiceLlmModel: string | null = null;
|
|
||||||
|
|
||||||
if (!resolvedVoiceLlmProvider) {
|
|
||||||
if (voiceLlmProviderExplicit) {
|
|
||||||
logger.error(
|
|
||||||
{ requestedVoiceLlmProvider, voiceLlmAvailability },
|
|
||||||
"No voice LLM provider available"
|
|
||||||
);
|
|
||||||
throw new Error("No voice LLM provider available");
|
|
||||||
}
|
|
||||||
logger.warn(
|
|
||||||
{ requestedVoiceLlmProvider, voiceLlmAvailability },
|
|
||||||
"No default voice LLM provider available; voice mode will be disabled until a provider is configured"
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
|
|
||||||
(definition) => definition.id === resolvedVoiceLlmProvider
|
|
||||||
);
|
|
||||||
if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
|
|
||||||
throw new Error(
|
|
||||||
`Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
|
|
||||||
resolvedVoiceLlmModel =
|
|
||||||
config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
requestedVoiceLlmProvider,
|
|
||||||
voiceLlmProviderExplicit,
|
|
||||||
resolvedVoiceLlmProvider,
|
|
||||||
resolvedVoiceLlmModeId,
|
|
||||||
resolvedVoiceLlmModel,
|
|
||||||
voiceLlmAvailability,
|
|
||||||
},
|
|
||||||
"Voice LLM provider reconciliation completed"
|
|
||||||
);
|
);
|
||||||
let wsServer: VoiceAssistantWebSocketServer | null = null;
|
let wsServer: VoiceAssistantWebSocketServer | null = null;
|
||||||
let voiceMcpBridgeManager: VoiceMcpSocketBridgeManager | null = null;
|
let voiceMcpBridgeManager: VoiceMcpSocketBridgeManager | null = null;
|
||||||
@@ -523,7 +432,7 @@ export async function createPaseoDaemon(
|
|||||||
agentStorage,
|
agentStorage,
|
||||||
paseoHome: config.paseoHome,
|
paseoHome: config.paseoHome,
|
||||||
callerAgentId,
|
callerAgentId,
|
||||||
enableVoiceTools: false,
|
voiceOnly: true,
|
||||||
resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
|
resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
|
||||||
resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
|
resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
|
||||||
logger,
|
logger,
|
||||||
@@ -555,10 +464,6 @@ export async function createPaseoDaemon(
|
|||||||
{ stt: sttService, tts: ttsService },
|
{ stt: sttService, tts: ttsService },
|
||||||
terminalManager,
|
terminalManager,
|
||||||
{
|
{
|
||||||
voiceLlmProvider: resolvedVoiceLlmProvider,
|
|
||||||
voiceLlmModeId: resolvedVoiceLlmModeId,
|
|
||||||
voiceLlmProviderExplicit,
|
|
||||||
voiceLlmModel: resolvedVoiceLlmModel,
|
|
||||||
voiceAgentMcpStdio: {
|
voiceAgentMcpStdio: {
|
||||||
command: voiceMcpBridgeCommand.command,
|
command: voiceMcpBridgeCommand.command,
|
||||||
baseArgs: [...voiceMcpBridgeCommand.baseArgs],
|
baseArgs: [...voiceMcpBridgeCommand.baseArgs],
|
||||||
|
|||||||
@@ -142,23 +142,29 @@ describe("daemon client E2E", () => {
|
|||||||
const agents = await ctx.client.fetchAgents();
|
const agents = await ctx.client.fetchAgents();
|
||||||
expect(Array.isArray(agents)).toBe(true);
|
expect(Array.isArray(agents)).toBe(true);
|
||||||
|
|
||||||
const voiceAgents = await ctx.client.fetchAgents({
|
const cwd = tmpCwd();
|
||||||
filter: { labels: { surface: "voice" } },
|
const created = await ctx.client.createAgent({
|
||||||
|
config: {
|
||||||
|
...getFullAccessConfig("codex"),
|
||||||
|
cwd,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
expect(Array.isArray(voiceAgents)).toBe(true);
|
|
||||||
|
|
||||||
await expect(ctx.client.setVoiceMode(true)).resolves.toMatchObject({
|
await expect(ctx.client.setVoiceMode(true, created.id)).resolves.toMatchObject({
|
||||||
enabled: true,
|
enabled: true,
|
||||||
|
agentId: created.id,
|
||||||
accepted: true,
|
accepted: true,
|
||||||
error: null,
|
error: null,
|
||||||
});
|
});
|
||||||
await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({
|
await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({
|
||||||
enabled: false,
|
enabled: false,
|
||||||
|
agentId: null,
|
||||||
accepted: true,
|
accepted: true,
|
||||||
error: null,
|
error: null,
|
||||||
});
|
});
|
||||||
|
|
||||||
await ctx.client.deleteAgent(randomUUID());
|
await ctx.client.deleteAgent(randomUUID());
|
||||||
|
rmSync(cwd, { recursive: true, force: true });
|
||||||
}, 30000);
|
}, 30000);
|
||||||
|
|
||||||
test("emits server_info on websocket connect", async () => {
|
test("emits server_info on websocket connect", async () => {
|
||||||
@@ -655,7 +661,14 @@ describe("daemon client E2E", () => {
|
|||||||
speechTest(
|
speechTest(
|
||||||
"voice mode buffers audio until isLast and emits transcription_result",
|
"voice mode buffers audio until isLast and emits transcription_result",
|
||||||
async () => {
|
async () => {
|
||||||
await ctx.client.setVoiceMode(true, randomUUID());
|
const voiceCwd = tmpCwd();
|
||||||
|
const voiceAgent = await ctx.client.createAgent({
|
||||||
|
config: {
|
||||||
|
...getFullAccessConfig("codex"),
|
||||||
|
cwd: voiceCwd,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await ctx.client.setVoiceMode(true, voiceAgent.id);
|
||||||
|
|
||||||
const transcription = waitForSignal(30_000, (resolve) => {
|
const transcription = waitForSignal(30_000, (resolve) => {
|
||||||
const unsubscribe = ctx.client.on("transcription_result", (message) => {
|
const unsubscribe = ctx.client.on("transcription_result", (message) => {
|
||||||
@@ -751,6 +764,7 @@ describe("daemon client E2E", () => {
|
|||||||
} finally {
|
} finally {
|
||||||
await Promise.allSettled([transcription, errorSignal]);
|
await Promise.allSettled([transcription, errorSignal]);
|
||||||
await ctx.client.setVoiceMode(false);
|
await ctx.client.setVoiceMode(false);
|
||||||
|
rmSync(voiceCwd, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
90_000
|
90_000
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { v4 as uuidv4 } from "uuid";
|
import { v4 as uuidv4 } from "uuid";
|
||||||
import { mkdir, stat } from "fs/promises";
|
import { stat } from "fs/promises";
|
||||||
import { exec } from "child_process";
|
import { exec } from "child_process";
|
||||||
import { promisify } from "util";
|
import { promisify } from "util";
|
||||||
import { join, resolve, sep } from "path";
|
import { join, resolve, sep } from "path";
|
||||||
@@ -58,6 +58,7 @@ import type {
|
|||||||
AgentPermissionResponse,
|
AgentPermissionResponse,
|
||||||
AgentPromptContentBlock,
|
AgentPromptContentBlock,
|
||||||
AgentPromptInput,
|
AgentPromptInput,
|
||||||
|
McpServerConfig,
|
||||||
AgentSessionConfig,
|
AgentSessionConfig,
|
||||||
AgentStreamEvent,
|
AgentStreamEvent,
|
||||||
AgentProvider,
|
AgentProvider,
|
||||||
@@ -191,7 +192,18 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(
|
|||||||
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
|
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
|
||||||
);
|
);
|
||||||
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
|
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
|
||||||
const VoiceAgentIdSchema = z.string().uuid();
|
const AgentIdSchema = z.string().uuid();
|
||||||
|
const VOICE_MCP_SERVER_NAME = "paseo_voice";
|
||||||
|
const VOICE_MODE_ENFORCED_MODE_ID: Partial<Record<AgentProvider, string>> = {
|
||||||
|
claude: "default",
|
||||||
|
codex: "read-only",
|
||||||
|
};
|
||||||
|
|
||||||
|
type VoiceModeBaseConfig = {
|
||||||
|
systemPrompt?: string;
|
||||||
|
mcpServers?: Record<string, McpServerConfig>;
|
||||||
|
modeId?: string;
|
||||||
|
};
|
||||||
|
|
||||||
interface AudioBufferState {
|
interface AudioBufferState {
|
||||||
chunks: Buffer[];
|
chunks: Buffer[];
|
||||||
@@ -214,10 +226,6 @@ export type SessionOptions = {
|
|||||||
tts: TextToSpeechProvider | null;
|
tts: TextToSpeechProvider | null;
|
||||||
terminalManager: TerminalManager | null;
|
terminalManager: TerminalManager | null;
|
||||||
voice?: {
|
voice?: {
|
||||||
voiceLlmProvider?: AgentProvider | null;
|
|
||||||
voiceLlmModeId?: string | null;
|
|
||||||
voiceLlmProviderExplicit?: boolean;
|
|
||||||
voiceLlmModel?: string | null;
|
|
||||||
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
|
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
|
||||||
};
|
};
|
||||||
voiceBridge?: {
|
voiceBridge?: {
|
||||||
@@ -376,10 +384,6 @@ export class Session {
|
|||||||
} | null = null;
|
} | null = null;
|
||||||
private readonly terminalManager: TerminalManager | null;
|
private readonly terminalManager: TerminalManager | null;
|
||||||
private terminalSubscriptions: Map<string, () => void> = new Map();
|
private terminalSubscriptions: Map<string, () => void> = new Map();
|
||||||
private readonly voiceLlmProvider: AgentProvider | null;
|
|
||||||
private readonly voiceLlmModeId: string | null;
|
|
||||||
private readonly voiceLlmProviderExplicit: boolean;
|
|
||||||
private readonly voiceLlmModel: string | null;
|
|
||||||
private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null;
|
private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null;
|
||||||
private readonly localSpeechModelsDir: string;
|
private readonly localSpeechModelsDir: string;
|
||||||
private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[];
|
private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[];
|
||||||
@@ -395,7 +399,8 @@ export class Session {
|
|||||||
private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
|
private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
|
||||||
private readonly ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
|
private readonly ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
|
||||||
private readonly removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
|
private readonly removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
|
||||||
private voiceAssistantAgentId: string | null = null;
|
private voiceModeAgentId: string | null = null;
|
||||||
|
private voiceModeBaseConfig: VoiceModeBaseConfig | null = null;
|
||||||
|
|
||||||
constructor(options: SessionOptions) {
|
constructor(options: SessionOptions) {
|
||||||
const {
|
const {
|
||||||
@@ -425,10 +430,6 @@ export class Session {
|
|||||||
this.agentStorage = agentStorage;
|
this.agentStorage = agentStorage;
|
||||||
this.createAgentMcpTransport = createAgentMcpTransport;
|
this.createAgentMcpTransport = createAgentMcpTransport;
|
||||||
this.terminalManager = terminalManager;
|
this.terminalManager = terminalManager;
|
||||||
this.voiceLlmProvider = voice?.voiceLlmProvider ?? null;
|
|
||||||
this.voiceLlmModeId = voice?.voiceLlmModeId ?? null;
|
|
||||||
this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false;
|
|
||||||
this.voiceLlmModel = voice?.voiceLlmModel ?? null;
|
|
||||||
this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null;
|
this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null;
|
||||||
const configuredModelsDir = dictation?.localModels?.modelsDir?.trim();
|
const configuredModelsDir = dictation?.localModels?.modelsDir?.trim();
|
||||||
this.localSpeechModelsDir =
|
this.localSpeechModelsDir =
|
||||||
@@ -896,7 +897,7 @@ export class Session {
|
|||||||
break;
|
break;
|
||||||
|
|
||||||
case "set_voice_mode":
|
case "set_voice_mode":
|
||||||
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId, msg.requestId);
|
await this.handleSetVoiceMode(msg.enabled, msg.agentId, msg.requestId);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case "send_agent_message_request":
|
case "send_agent_message_request":
|
||||||
@@ -1276,108 +1277,216 @@ export class Session {
|
|||||||
*/
|
*/
|
||||||
private async handleSetVoiceMode(
|
private async handleSetVoiceMode(
|
||||||
enabled: boolean,
|
enabled: boolean,
|
||||||
voiceAgentId?: string,
|
agentId?: string,
|
||||||
requestId?: string
|
requestId?: string
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (enabled) {
|
try {
|
||||||
let normalizedVoiceAgentId: string | null = null;
|
if (enabled) {
|
||||||
if (voiceAgentId && voiceAgentId.trim().length > 0) {
|
const normalizedAgentId = this.parseVoiceTargetAgentId(
|
||||||
normalizedVoiceAgentId = this.parseVoiceAgentId(
|
agentId ?? "",
|
||||||
voiceAgentId,
|
|
||||||
"set_voice_mode"
|
"set_voice_mode"
|
||||||
);
|
);
|
||||||
} else {
|
|
||||||
normalizedVoiceAgentId = await this.findExistingVoiceAssistantAgentId();
|
if (
|
||||||
|
this.isVoiceMode &&
|
||||||
|
this.voiceModeAgentId &&
|
||||||
|
this.voiceModeAgentId !== normalizedAgentId
|
||||||
|
) {
|
||||||
|
await this.disableVoiceModeForActiveAgent(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.isVoiceMode || this.voiceModeAgentId !== normalizedAgentId) {
|
||||||
|
const refreshedAgentId = await this.enableVoiceModeForAgent(normalizedAgentId);
|
||||||
|
this.voiceModeAgentId = refreshedAgentId;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.isVoiceMode = true;
|
||||||
|
this.sessionLogger.info(
|
||||||
|
{
|
||||||
|
agentId: this.voiceModeAgentId,
|
||||||
|
},
|
||||||
|
"Voice mode enabled for existing agent"
|
||||||
|
);
|
||||||
|
if (requestId) {
|
||||||
|
this.emit({
|
||||||
|
type: "set_voice_mode_response",
|
||||||
|
payload: {
|
||||||
|
requestId,
|
||||||
|
enabled: true,
|
||||||
|
agentId: this.voiceModeAgentId,
|
||||||
|
accepted: true,
|
||||||
|
error: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.isVoiceMode = true;
|
await this.disableVoiceModeForActiveAgent(true);
|
||||||
this.voiceAssistantAgentId = normalizedVoiceAgentId;
|
this.isVoiceMode = false;
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info("Voice mode disabled");
|
||||||
|
if (requestId) {
|
||||||
|
this.emit({
|
||||||
|
type: "set_voice_mode_response",
|
||||||
|
payload: {
|
||||||
|
requestId,
|
||||||
|
enabled: false,
|
||||||
|
agentId: null,
|
||||||
|
accepted: true,
|
||||||
|
error: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Failed to set voice mode";
|
||||||
|
this.sessionLogger.error(
|
||||||
{
|
{
|
||||||
voiceAssistantAgentId: this.voiceAssistantAgentId,
|
err: error,
|
||||||
resumedVoiceAgent: Boolean(normalizedVoiceAgentId),
|
enabled,
|
||||||
providedVoiceAgentId: Boolean(voiceAgentId?.trim()),
|
requestedAgentId: agentId ?? null,
|
||||||
},
|
},
|
||||||
"Voice mode enabled (agent-backed)"
|
"set_voice_mode failed"
|
||||||
);
|
);
|
||||||
if (requestId) {
|
if (requestId) {
|
||||||
this.emit({
|
this.emit({
|
||||||
type: "set_voice_mode_response",
|
type: "set_voice_mode_response",
|
||||||
payload: {
|
payload: {
|
||||||
requestId,
|
requestId,
|
||||||
enabled: true,
|
enabled: this.isVoiceMode,
|
||||||
voiceAgentId: this.voiceAssistantAgentId,
|
agentId: this.voiceModeAgentId,
|
||||||
accepted: true,
|
accepted: false,
|
||||||
error: null,
|
error: errorMessage,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
return;
|
throw error;
|
||||||
}
|
|
||||||
|
|
||||||
this.isVoiceMode = false;
|
|
||||||
const disabledVoiceAgentId = this.voiceAssistantAgentId;
|
|
||||||
if (disabledVoiceAgentId && this.removeVoiceMcpSocketForAgent) {
|
|
||||||
try {
|
|
||||||
await this.removeVoiceMcpSocketForAgent(disabledVoiceAgentId);
|
|
||||||
} catch (error) {
|
|
||||||
this.sessionLogger.warn(
|
|
||||||
{ err: error, voiceAssistantAgentId: disabledVoiceAgentId },
|
|
||||||
"Failed to remove voice MCP socket bridge on disable"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.sessionLogger.info(
|
|
||||||
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
|
|
||||||
"Voice mode disabled (agent-backed)"
|
|
||||||
);
|
|
||||||
if (requestId) {
|
|
||||||
this.emit({
|
|
||||||
type: "set_voice_mode_response",
|
|
||||||
payload: {
|
|
||||||
requestId,
|
|
||||||
enabled: false,
|
|
||||||
voiceAgentId: this.voiceAssistantAgentId,
|
|
||||||
accepted: true,
|
|
||||||
error: null,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private parseVoiceAgentId(rawId: string, source: string): string {
|
private parseVoiceTargetAgentId(rawId: string, source: string): string {
|
||||||
const parsed = VoiceAgentIdSchema.safeParse(rawId.trim());
|
const parsed = AgentIdSchema.safeParse(rawId.trim());
|
||||||
if (!parsed.success) {
|
if (!parsed.success) {
|
||||||
throw new Error(`${source}: voiceAgentId must be a UUID`);
|
throw new Error(`${source}: agentId must be a UUID`);
|
||||||
}
|
}
|
||||||
return parsed.data;
|
return parsed.data;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async findExistingVoiceAssistantAgentId(): Promise<string | null> {
|
private cloneMcpServers(
|
||||||
const scopedCandidates = await this.listAgentPayloads({
|
servers: Record<string, McpServerConfig> | undefined
|
||||||
labels: {
|
): Record<string, McpServerConfig> | undefined {
|
||||||
surface: "voice",
|
if (!servers) {
|
||||||
voiceClientId: this.clientId,
|
return undefined;
|
||||||
},
|
|
||||||
});
|
|
||||||
const fallbackCandidates = await this.listAgentPayloads({
|
|
||||||
labels: { surface: "voice" },
|
|
||||||
});
|
|
||||||
const available = [
|
|
||||||
...scopedCandidates,
|
|
||||||
...fallbackCandidates.filter(
|
|
||||||
(candidate) => !scopedCandidates.some((scoped) => scoped.id === candidate.id)
|
|
||||||
),
|
|
||||||
].filter((agent) => !agent.archivedAt && agent.labels?.ui !== "false");
|
|
||||||
if (available.length === 0) {
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
available.sort((left, right) => {
|
return JSON.parse(JSON.stringify(servers)) as Record<string, McpServerConfig>;
|
||||||
const leftTime = Date.parse(left.updatedAt);
|
}
|
||||||
const rightTime = Date.parse(right.updatedAt);
|
|
||||||
return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0);
|
private buildVoiceModeSystemPrompt(existing?: string): string {
|
||||||
|
const chunks = [existing?.trim(), VOICE_AGENT_SYSTEM_INSTRUCTION].filter(
|
||||||
|
(value): value is string => Boolean(value && value.length > 0)
|
||||||
|
);
|
||||||
|
return chunks.join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildVoiceModeMcpServers(
|
||||||
|
existing: Record<string, McpServerConfig> | undefined,
|
||||||
|
socketPath: string
|
||||||
|
): Record<string, McpServerConfig> {
|
||||||
|
const mcpStdio = this.voiceAgentMcpStdio;
|
||||||
|
if (!mcpStdio) {
|
||||||
|
throw new Error("Voice MCP stdio bridge is not configured");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...(existing ?? {}),
|
||||||
|
[VOICE_MCP_SERVER_NAME]: buildVoiceAgentMcpServerConfig({
|
||||||
|
command: mcpStdio.command,
|
||||||
|
baseArgs: mcpStdio.baseArgs,
|
||||||
|
socketPath,
|
||||||
|
env: mcpStdio.env,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private getVoiceModeEnforcedModeId(provider: AgentProvider): string | undefined {
|
||||||
|
return VOICE_MODE_ENFORCED_MODE_ID[provider];
|
||||||
|
}
|
||||||
|
|
||||||
|
private async enableVoiceModeForAgent(agentId: string): Promise<string> {
|
||||||
|
const ensureVoiceSocket = this.ensureVoiceMcpSocketForAgent;
|
||||||
|
if (!ensureVoiceSocket) {
|
||||||
|
throw new Error("Voice MCP socket bridge is not configured");
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = await this.ensureAgentLoaded(agentId);
|
||||||
|
|
||||||
|
const socketPath = await ensureVoiceSocket(agentId);
|
||||||
|
this.registerVoiceBridgeForAgent(agentId);
|
||||||
|
|
||||||
|
const baseConfig: VoiceModeBaseConfig = {
|
||||||
|
systemPrompt: existing.config.systemPrompt,
|
||||||
|
mcpServers: this.cloneMcpServers(existing.config.mcpServers),
|
||||||
|
modeId: existing.config.modeId,
|
||||||
|
};
|
||||||
|
this.voiceModeBaseConfig = baseConfig;
|
||||||
|
const voiceModeId = this.getVoiceModeEnforcedModeId(existing.config.provider);
|
||||||
|
const refreshOverrides: Partial<AgentSessionConfig> = {
|
||||||
|
systemPrompt: this.buildVoiceModeSystemPrompt(baseConfig.systemPrompt),
|
||||||
|
mcpServers: this.buildVoiceModeMcpServers(baseConfig.mcpServers, socketPath),
|
||||||
|
...(voiceModeId ? { modeId: voiceModeId } : {}),
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const refreshed = await this.agentManager.refreshAgentFromPersistence(
|
||||||
|
agentId,
|
||||||
|
refreshOverrides
|
||||||
|
);
|
||||||
|
return refreshed.id;
|
||||||
|
} catch (error) {
|
||||||
|
this.unregisterVoiceSpeakHandler?.(agentId);
|
||||||
|
this.unregisterVoiceCallerContext?.(agentId);
|
||||||
|
await this.removeVoiceMcpSocketForAgent?.(agentId).catch(() => undefined);
|
||||||
|
this.voiceModeBaseConfig = null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async disableVoiceModeForActiveAgent(
|
||||||
|
restoreAgentConfig: boolean
|
||||||
|
): Promise<void> {
|
||||||
|
const agentId = this.voiceModeAgentId;
|
||||||
|
if (!agentId) {
|
||||||
|
this.voiceModeBaseConfig = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.unregisterVoiceSpeakHandler?.(agentId);
|
||||||
|
this.unregisterVoiceCallerContext?.(agentId);
|
||||||
|
await this.removeVoiceMcpSocketForAgent?.(agentId).catch((error) => {
|
||||||
|
this.sessionLogger.warn(
|
||||||
|
{ err: error, agentId },
|
||||||
|
"Failed to remove voice MCP socket bridge on disable"
|
||||||
|
);
|
||||||
});
|
});
|
||||||
return available[0]?.id ?? null;
|
|
||||||
|
if (restoreAgentConfig && this.voiceModeBaseConfig) {
|
||||||
|
const baseConfig = this.voiceModeBaseConfig;
|
||||||
|
try {
|
||||||
|
await this.agentManager.refreshAgentFromPersistence(agentId, {
|
||||||
|
systemPrompt: baseConfig.systemPrompt,
|
||||||
|
mcpServers: this.cloneMcpServers(baseConfig.mcpServers),
|
||||||
|
modeId: baseConfig.modeId,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
this.sessionLogger.warn(
|
||||||
|
{ err: error, agentId },
|
||||||
|
"Failed to restore agent config while disabling voice mode"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.voiceModeBaseConfig = null;
|
||||||
|
this.voiceModeAgentId = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -4501,101 +4610,11 @@ export class Session {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async ensureVoiceAssistantAgent(): Promise<string> {
|
|
||||||
const ensureVoiceSocket = this.ensureVoiceMcpSocketForAgent;
|
|
||||||
if (!ensureVoiceSocket) {
|
|
||||||
throw new Error("Voice MCP socket bridge is not configured");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.voiceAssistantAgentId) {
|
|
||||||
const existing = this.agentManager.getAgent(this.voiceAssistantAgentId);
|
|
||||||
if (existing) {
|
|
||||||
await ensureVoiceSocket(existing.id);
|
|
||||||
this.registerVoiceBridgeForAgent(existing.id);
|
|
||||||
return existing.id;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const hydrated = await this.ensureAgentLoaded(this.voiceAssistantAgentId);
|
|
||||||
await ensureVoiceSocket(hydrated.id);
|
|
||||||
this.voiceAssistantAgentId = hydrated.id;
|
|
||||||
this.registerVoiceBridgeForAgent(hydrated.id);
|
|
||||||
return hydrated.id;
|
|
||||||
} catch (error) {
|
|
||||||
this.sessionLogger.debug(
|
|
||||||
{ err: error, voiceAssistantAgentId: this.voiceAssistantAgentId },
|
|
||||||
"Voice assistant agent not found in active/persisted state; creating new session"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const provider = this.voiceLlmProvider;
|
|
||||||
if (!provider) {
|
|
||||||
throw new Error("Voice LLM provider is not configured");
|
|
||||||
}
|
|
||||||
const voiceAgentId = this.voiceAssistantAgentId ?? uuidv4();
|
|
||||||
const cwd = join(this.paseoHome, "voice-agent-workspace");
|
|
||||||
await mkdir(cwd, { recursive: true });
|
|
||||||
|
|
||||||
const mcpStdio = this.voiceAgentMcpStdio;
|
|
||||||
if (!mcpStdio) {
|
|
||||||
throw new Error("Voice MCP stdio bridge is not configured");
|
|
||||||
}
|
|
||||||
const socketPath = await ensureVoiceSocket(voiceAgentId);
|
|
||||||
|
|
||||||
const model = this.voiceLlmModel?.trim() || undefined;
|
|
||||||
const config: AgentSessionConfig = {
|
|
||||||
provider,
|
|
||||||
cwd,
|
|
||||||
systemPrompt: VOICE_AGENT_SYSTEM_INSTRUCTION,
|
|
||||||
modeId: this.voiceLlmModeId ?? "default",
|
|
||||||
...(model ? { model } : {}),
|
|
||||||
mcpServers: {
|
|
||||||
paseo: buildVoiceAgentMcpServerConfig({
|
|
||||||
command: mcpStdio.command,
|
|
||||||
baseArgs: mcpStdio.baseArgs,
|
|
||||||
socketPath,
|
|
||||||
env: mcpStdio.env,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
const created = await this.agentManager.createAgent(config, voiceAgentId, {
|
|
||||||
labels: {
|
|
||||||
surface: "voice",
|
|
||||||
ui: "true",
|
|
||||||
voiceClientId: this.clientId,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
this.voiceAssistantAgentId = created.id;
|
|
||||||
|
|
||||||
try {
|
|
||||||
this.registerVoiceBridgeForAgent(created.id);
|
|
||||||
} catch (error) {
|
|
||||||
this.unregisterVoiceSpeakHandler?.(created.id);
|
|
||||||
this.unregisterVoiceCallerContext?.(created.id);
|
|
||||||
await this.removeVoiceMcpSocketForAgent?.(created.id).catch(() => undefined);
|
|
||||||
await this.agentManager.closeAgent(created.id).catch(() => undefined);
|
|
||||||
this.voiceAssistantAgentId = null;
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.sessionLogger.info(
|
|
||||||
{
|
|
||||||
voiceAssistantAgentId: created.id,
|
|
||||||
provider,
|
|
||||||
model: model ?? null,
|
|
||||||
providerExplicit: this.voiceLlmProviderExplicit,
|
|
||||||
},
|
|
||||||
"Voice assistant agent initialized"
|
|
||||||
);
|
|
||||||
return created.id;
|
|
||||||
}
|
|
||||||
|
|
||||||
private registerVoiceBridgeForAgent(agentId: string): void {
|
private registerVoiceBridgeForAgent(agentId: string): void {
|
||||||
this.registerVoiceSpeakHandler?.(agentId, async ({ text, signal }) => {
|
this.registerVoiceSpeakHandler?.(agentId, async ({ text, signal }) => {
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{
|
{
|
||||||
voiceAssistantAgentId: agentId,
|
agentId,
|
||||||
textLength: text.length,
|
textLength: text.length,
|
||||||
preview: text.slice(0, 160),
|
preview: text.slice(0, 160),
|
||||||
},
|
},
|
||||||
@@ -4609,7 +4628,7 @@ export class Session {
|
|||||||
true
|
true
|
||||||
);
|
);
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{ voiceAssistantAgentId: agentId, textLength: text.length },
|
{ agentId, textLength: text.length },
|
||||||
"Voice speak tool call finished playback"
|
"Voice speak tool call finished playback"
|
||||||
);
|
);
|
||||||
this.emit({
|
this.emit({
|
||||||
@@ -4625,7 +4644,7 @@ export class Session {
|
|||||||
|
|
||||||
this.registerVoiceCallerContext?.(agentId, {
|
this.registerVoiceCallerContext?.(agentId, {
|
||||||
childAgentDefaultLabels: { ui: "true" },
|
childAgentDefaultLabels: { ui: "true" },
|
||||||
allowCustomCwd: true,
|
allowCustomCwd: false,
|
||||||
enableVoiceTools: true,
|
enableVoiceTools: true,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -4643,14 +4662,16 @@ export class Session {
|
|||||||
return isVoicePermissionAllowed(request);
|
return isVoicePermissionAllowed(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async processWithVoiceAgent(userText: string): Promise<void> {
|
private async processWithVoiceAgent(
|
||||||
const agentId = await this.ensureVoiceAssistantAgent();
|
agentId: string,
|
||||||
|
userText: string
|
||||||
|
): Promise<void> {
|
||||||
|
await this.ensureAgentLoaded(agentId);
|
||||||
await this.interruptAgentIfRunning(agentId);
|
await this.interruptAgentIfRunning(agentId);
|
||||||
|
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{
|
{
|
||||||
voiceAssistantAgentId: agentId,
|
agentId,
|
||||||
userTextLength: userText.length,
|
userTextLength: userText.length,
|
||||||
userTextPreview: userText.slice(0, 160),
|
userTextPreview: userText.slice(0, 160),
|
||||||
},
|
},
|
||||||
@@ -4670,7 +4691,7 @@ export class Session {
|
|||||||
if (event.item.type === "tool_call" && typeof event.item.name === "string") {
|
if (event.item.type === "tool_call" && typeof event.item.name === "string") {
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{
|
{
|
||||||
voiceAssistantAgentId: agentId,
|
agentId,
|
||||||
toolName: event.item.name,
|
toolName: event.item.name,
|
||||||
callId: event.item.callId,
|
callId: event.item.callId,
|
||||||
},
|
},
|
||||||
@@ -4689,11 +4710,12 @@ export class Session {
|
|||||||
} else {
|
} else {
|
||||||
await this.agentManager.respondToPermission(agentId, event.request.id, {
|
await this.agentManager.respondToPermission(agentId, event.request.id, {
|
||||||
behavior: "deny",
|
behavior: "deny",
|
||||||
message: "Voice assistant policy only allows MCP paseo tools.",
|
message: "Voice assistant policy only allows the speak tool.",
|
||||||
interrupt: true,
|
interrupt: true,
|
||||||
});
|
});
|
||||||
throw new Error(
|
this.sessionLogger.warn(
|
||||||
`Voice assistant denied non-MCP tool request: ${event.request.name}`
|
{ agentId, requestName: event.request.name, requestId: event.request.id },
|
||||||
|
"Voice assistant denied non-speak permission request"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -4701,12 +4723,12 @@ export class Session {
|
|||||||
|
|
||||||
if (!sawSpeakToolCall) {
|
if (!sawSpeakToolCall) {
|
||||||
this.sessionLogger.warn(
|
this.sessionLogger.warn(
|
||||||
{ voiceAssistantAgentId: agentId },
|
{ agentId },
|
||||||
"Voice agent turn completed without speak tool call; no audio playback emitted"
|
"Voice agent turn completed without speak tool call; no audio playback emitted"
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{ voiceAssistantAgentId: agentId },
|
{ agentId },
|
||||||
"Voice agent turn included speak tool call"
|
"Voice agent turn included speak tool call"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -4721,11 +4743,18 @@ export class Session {
|
|||||||
this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled");
|
this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const agentId = this.voiceModeAgentId;
|
||||||
|
if (!agentId) {
|
||||||
|
this.sessionLogger.warn(
|
||||||
|
"Ignoring processVoiceTurn call because no agent is currently voice-enabled"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
const normalized = (latestUserText ?? "").trim();
|
const normalized = (latestUserText ?? "").trim();
|
||||||
if (!normalized) {
|
if (!normalized) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await this.processWithVoiceAgent(normalized);
|
await this.processWithVoiceAgent(agentId, normalized);
|
||||||
} finally {
|
} finally {
|
||||||
this.turnIndex++;
|
this.turnIndex++;
|
||||||
this.currentStreamPromise = null;
|
this.currentStreamPromise = null;
|
||||||
@@ -4980,21 +5009,8 @@ export class Session {
|
|||||||
this.agentTools = null;
|
this.agentTools = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.voiceAssistantAgentId) {
|
await this.disableVoiceModeForActiveAgent(true);
|
||||||
try {
|
this.isVoiceMode = false;
|
||||||
await this.agentManager.closeAgent(this.voiceAssistantAgentId);
|
|
||||||
} catch (error) {
|
|
||||||
this.sessionLogger.warn(
|
|
||||||
{ err: error, voiceAssistantAgentId: this.voiceAssistantAgentId },
|
|
||||||
"Failed to close voice assistant agent"
|
|
||||||
);
|
|
||||||
} finally {
|
|
||||||
this.unregisterVoiceSpeakHandler?.(this.voiceAssistantAgentId);
|
|
||||||
this.unregisterVoiceCallerContext?.(this.voiceAssistantAgentId);
|
|
||||||
await this.removeVoiceMcpSocketForAgent?.(this.voiceAssistantAgentId).catch(() => undefined);
|
|
||||||
this.voiceAssistantAgentId = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unsubscribe from all terminals
|
// Unsubscribe from all terminals
|
||||||
for (const unsubscribe of this.terminalSubscriptions.values()) {
|
for (const unsubscribe of this.terminalSubscriptions.values()) {
|
||||||
|
|||||||
+13
-3
@@ -1,8 +1,7 @@
|
|||||||
import { describe, expect, test } from "vitest";
|
import { describe, expect, test } from "vitest";
|
||||||
import { mkdtempSync } from "node:fs";
|
import { mkdtempSync, rmSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { randomUUID } from "node:crypto";
|
|
||||||
import pino from "pino";
|
import pino from "pino";
|
||||||
|
|
||||||
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
|
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
|
||||||
@@ -191,7 +190,17 @@ describe("speech models (download E2E)", () => {
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
await ctx.client.setVoiceMode(true, randomUUID());
|
const voiceCwd = mkdtempSync(path.join(tmpdir(), "speech-download-voice-agent-"));
|
||||||
|
const voiceAgent = await ctx.client.createAgent({
|
||||||
|
config: {
|
||||||
|
provider: "codex",
|
||||||
|
cwd: voiceCwd,
|
||||||
|
modeId: "full-access",
|
||||||
|
model: "gpt-5.1-codex-mini",
|
||||||
|
thinkingOptionId: "low",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await ctx.client.setVoiceMode(true, voiceAgent.id);
|
||||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||||
const isLast = offset + chunkBytes >= pcm16.length;
|
const isLast = offset + chunkBytes >= pcm16.length;
|
||||||
@@ -202,6 +211,7 @@ describe("speech models (download E2E)", () => {
|
|||||||
expect(voiceText).toContain("voice note");
|
expect(voiceText).toContain("voice note");
|
||||||
}
|
}
|
||||||
await ctx.client.setVoiceMode(false);
|
await ctx.client.setVoiceMode(false);
|
||||||
|
rmSync(voiceCwd, { recursive: true, force: true });
|
||||||
|
|
||||||
// Streaming TTS: generate locally from downloaded model and validate chunking.
|
// Streaming TTS: generate locally from downloaded model and validate chunking.
|
||||||
const ttsText = "This is a voice note.";
|
const ttsText = "This is a voice note.";
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
||||||
import { randomUUID } from "node:crypto";
|
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { readFile } from "node:fs/promises";
|
import { mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import pino from "pino";
|
||||||
|
|
||||||
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
|
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
|
||||||
|
import { getFullAccessConfig } from "./daemon-e2e/agent-configs.js";
|
||||||
|
import { OpenAITTS } from "./speech/providers/openai/tts.js";
|
||||||
|
|
||||||
const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
|
const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
|
||||||
const shouldRun =
|
const shouldRun =
|
||||||
@@ -69,61 +72,94 @@ function waitForSignal<T>(
|
|||||||
test(
|
test(
|
||||||
"routes voice turns through local agent speak tool",
|
"routes voice turns through local agent speak tool",
|
||||||
async () => {
|
async () => {
|
||||||
await ctx.client.setVoiceMode(true, randomUUID());
|
const logger = pino({ level: "silent" });
|
||||||
|
const ttsProvider = new OpenAITTS(
|
||||||
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
|
{
|
||||||
const offAudio = ctx.client.on("audio_output", (msg) => {
|
apiKey: openaiApiKey!,
|
||||||
if (msg.type !== "audio_output") return;
|
responseFormat: "pcm",
|
||||||
resolve({ chunkId: msg.payload.id });
|
voice: "alloy",
|
||||||
});
|
},
|
||||||
const offError = ctx.client.on("activity_log", (msg) => {
|
logger
|
||||||
if (msg.type !== "activity_log") return;
|
|
||||||
if (msg.payload.type !== "error") return;
|
|
||||||
reject(new Error(String(msg.payload.content)));
|
|
||||||
});
|
|
||||||
return () => {
|
|
||||||
offAudio();
|
|
||||||
offError();
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
const assistantLogPromise = waitForSignal<string>(120000, (resolve, reject) => {
|
|
||||||
const offLog = ctx.client.on("activity_log", (msg) => {
|
|
||||||
if (msg.type !== "activity_log") return;
|
|
||||||
if (msg.payload.type !== "assistant") return;
|
|
||||||
const content = String(msg.payload.content ?? "");
|
|
||||||
if (!content.trim()) return;
|
|
||||||
resolve(content);
|
|
||||||
});
|
|
||||||
const offError = ctx.client.on("activity_log", (msg) => {
|
|
||||||
if (msg.type !== "activity_log") return;
|
|
||||||
if (msg.payload.type !== "error") return;
|
|
||||||
reject(new Error(String(msg.payload.content)));
|
|
||||||
});
|
|
||||||
return () => {
|
|
||||||
offLog();
|
|
||||||
offError();
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
const fixturePath = path.resolve(
|
|
||||||
process.cwd(),
|
|
||||||
"..",
|
|
||||||
"app",
|
|
||||||
"e2e",
|
|
||||||
"fixtures",
|
|
||||||
"recording.wav"
|
|
||||||
);
|
);
|
||||||
const wav = await readFile(fixturePath);
|
|
||||||
await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true);
|
|
||||||
|
|
||||||
const [{ chunkId }, assistantText] = await Promise.all([
|
const voiceCwd = mkdtempSync(path.join(tmpdir(), "voice-local-agent-"));
|
||||||
audioPromise,
|
const targetAgent = await ctx.client.createAgent({
|
||||||
assistantLogPromise,
|
config: {
|
||||||
|
...getFullAccessConfig("codex"),
|
||||||
|
cwd: voiceCwd,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const voiceMode = await ctx.client.setVoiceMode(true, targetAgent.id);
|
||||||
|
expect(voiceMode.accepted).toBe(true);
|
||||||
|
|
||||||
|
const transcriptionPromise = waitForSignal<string>(120000, (resolve, reject) => {
|
||||||
|
const offTranscript = ctx.client.on("transcription_result", (msg) => {
|
||||||
|
if (msg.type !== "transcription_result") return;
|
||||||
|
const text = String(msg.payload.text ?? "").trim();
|
||||||
|
if (!text) return;
|
||||||
|
resolve(text);
|
||||||
|
});
|
||||||
|
const offError = ctx.client.on("activity_log", (msg) => {
|
||||||
|
if (msg.type !== "activity_log") return;
|
||||||
|
if (msg.payload.type !== "error") return;
|
||||||
|
reject(new Error(String(msg.payload.content)));
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
offTranscript();
|
||||||
|
offError();
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const speakToolPromise = waitForSignal<string>(120000, (resolve, reject) => {
|
||||||
|
const offStream = ctx.client.on("agent_stream", (msg) => {
|
||||||
|
if (msg.type !== "agent_stream") return;
|
||||||
|
if (msg.payload.event.type !== "timeline") return;
|
||||||
|
const item = msg.payload.event.item;
|
||||||
|
if (item.type !== "tool_call") return;
|
||||||
|
const name = String(item.name ?? "");
|
||||||
|
if (!name.toLowerCase().includes("speak")) return;
|
||||||
|
resolve(name);
|
||||||
|
});
|
||||||
|
const offError = ctx.client.on("activity_log", (msg) => {
|
||||||
|
if (msg.type !== "activity_log") return;
|
||||||
|
if (msg.payload.type !== "error") return;
|
||||||
|
reject(new Error(String(msg.payload.content)));
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
offStream();
|
||||||
|
offError();
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const inputSpeech = await ttsProvider.synthesizeSpeech(
|
||||||
|
"Use the speak tool and say exactly local agent check."
|
||||||
|
);
|
||||||
|
const buffers: Buffer[] = [];
|
||||||
|
for await (const chunk of inputSpeech.stream as AsyncIterable<unknown>) {
|
||||||
|
buffers.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as ArrayBufferLike));
|
||||||
|
}
|
||||||
|
const pcm = Buffer.concat(buffers);
|
||||||
|
const chunkBytes = 4800;
|
||||||
|
for (let offset = 0; offset < pcm.length; offset += chunkBytes) {
|
||||||
|
const chunk = pcm.subarray(offset, Math.min(pcm.length, offset + chunkBytes));
|
||||||
|
const isLast = offset + chunkBytes >= pcm.length;
|
||||||
|
await ctx.client.sendVoiceAudioChunk(
|
||||||
|
chunk.toString("base64"),
|
||||||
|
"audio/pcm;rate=24000;bits=16",
|
||||||
|
isLast
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const [transcript, speakToolName] = await Promise.all([
|
||||||
|
transcriptionPromise,
|
||||||
|
speakToolPromise,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
expect(chunkId.length).toBeGreaterThan(0);
|
await ctx.client.setVoiceMode(false).catch(() => undefined);
|
||||||
expect(assistantText.trim().length).toBeGreaterThan(0);
|
rmSync(voiceCwd, { recursive: true, force: true });
|
||||||
|
|
||||||
|
expect(transcript.length).toBeGreaterThan(0);
|
||||||
|
expect(speakToolName.toLowerCase()).toContain("speak");
|
||||||
|
|
||||||
const agents = await ctx.client.fetchAgents();
|
const agents = await ctx.client.fetchAgents();
|
||||||
expect(
|
expect(
|
||||||
|
|||||||
@@ -21,13 +21,13 @@ describe("isVoicePermissionAllowed", () => {
|
|||||||
expect(result).toBe(true);
|
expect(result).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("allows explicit MCP/paseo tool names", () => {
|
test("denies explicit MCP/paseo tool names", () => {
|
||||||
expect(
|
expect(
|
||||||
isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))
|
isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))
|
||||||
).toBe(true);
|
).toBe(false);
|
||||||
expect(
|
expect(
|
||||||
isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))
|
isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))
|
||||||
).toBe(true);
|
).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("denies non-tool permission kinds", () => {
|
test("denies non-tool permission kinds", () => {
|
||||||
@@ -37,12 +37,12 @@ describe("isVoicePermissionAllowed", () => {
|
|||||||
expect(result).toBe(false);
|
expect(result).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("allows codextool only when metadata references MCP/paseo", () => {
|
test("allows codextool only when metadata references speak", () => {
|
||||||
const allowed = isVoicePermissionAllowed(
|
const allowed = isVoicePermissionAllowed(
|
||||||
buildRequest({
|
buildRequest({
|
||||||
name: "codextool",
|
name: "codextool",
|
||||||
metadata: {
|
metadata: {
|
||||||
questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }],
|
questions: [{ question: "Allow codextool to call speak for user feedback?" }],
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
|
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
|
||||||
|
|
||||||
const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"];
|
const ALLOWED_TOKEN_SET = ["speak"];
|
||||||
const DENIED_TOKEN_SET = [
|
const DENIED_TOKEN_SET = [
|
||||||
|
"mcp",
|
||||||
|
"paseo",
|
||||||
"bash",
|
"bash",
|
||||||
"shell",
|
"shell",
|
||||||
"terminal",
|
"terminal",
|
||||||
@@ -27,10 +29,7 @@ function stringifyMetadata(metadata: unknown): string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** Voice assistant policy: only allow the speak tool. */
|
||||||
* Voice assistant policy: allow only MCP/paseo/speak tool requests.
|
|
||||||
* All non-tool permission requests are denied.
|
|
||||||
*/
|
|
||||||
export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean {
|
export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean {
|
||||||
if (request.kind !== "tool") {
|
if (request.kind !== "tool") {
|
||||||
return false;
|
return false;
|
||||||
@@ -44,9 +43,6 @@ export function isVoicePermissionAllowed(request: AgentPermissionRequest): boole
|
|||||||
if (normalizedName === "speak") {
|
if (normalizedName === "speak") {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (normalizedName !== "codextool") {
|
if (normalizedName !== "codextool") {
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -1,12 +1,15 @@
|
|||||||
import { describe, test, expect, beforeAll, afterAll } from "vitest";
|
import { describe, test, expect, beforeAll, afterAll } from "vitest";
|
||||||
import { randomUUID } from "node:crypto";
|
|
||||||
import { Readable } from "node:stream";
|
import { Readable } from "node:stream";
|
||||||
|
import { mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
import pino from "pino";
|
import pino from "pino";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
createDaemonTestContext,
|
createDaemonTestContext,
|
||||||
type DaemonTestContext,
|
type DaemonTestContext,
|
||||||
} from "./test-utils/index.js";
|
} from "./test-utils/index.js";
|
||||||
|
import { getFullAccessConfig, type AgentProvider } from "./daemon-e2e/agent-configs.js";
|
||||||
import { OpenAITTS } from "./speech/providers/openai/tts.js";
|
import { OpenAITTS } from "./speech/providers/openai/tts.js";
|
||||||
import { OpenAISTT } from "./speech/providers/openai/stt.js";
|
import { OpenAISTT } from "./speech/providers/openai/stt.js";
|
||||||
import { STTManager } from "./agent/stt-manager.js";
|
import { STTManager } from "./agent/stt-manager.js";
|
||||||
@@ -88,8 +91,6 @@ describe("voice roundtrip e2e", () => {
|
|||||||
voiceTts: { provider: "openai", explicit: true },
|
voiceTts: { provider: "openai", explicit: true },
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
voiceLlmProvider: "claude",
|
|
||||||
voiceLlmProviderExplicit: true,
|
|
||||||
});
|
});
|
||||||
}, 60000);
|
}, 60000);
|
||||||
|
|
||||||
@@ -97,9 +98,10 @@ describe("voice roundtrip e2e", () => {
|
|||||||
await ctx.cleanup();
|
await ctx.cleanup();
|
||||||
}, 60000);
|
}, 60000);
|
||||||
|
|
||||||
speechTest(
|
for (const targetProvider of ["claude", "codex"] as const satisfies AgentProvider[]) {
|
||||||
"full roundtrip: voice input audio -> voice agent -> output audio -> transcribed output",
|
speechTest(
|
||||||
async () => {
|
`full roundtrip (${targetProvider}): voice input audio -> voice agent -> output audio -> transcribed output`,
|
||||||
|
async () => {
|
||||||
const logger = pino({ level: "silent" });
|
const logger = pino({ level: "silent" });
|
||||||
const ttsProvider = new OpenAITTS(
|
const ttsProvider = new OpenAITTS(
|
||||||
{
|
{
|
||||||
@@ -118,7 +120,20 @@ describe("voice roundtrip e2e", () => {
|
|||||||
);
|
);
|
||||||
const sttOutput = new STTManager("voice-roundtrip-e2e", logger, sttProvider);
|
const sttOutput = new STTManager("voice-roundtrip-e2e", logger, sttProvider);
|
||||||
|
|
||||||
const voiceAgentId = randomUUID();
|
const voiceCwd = mkdtempSync(
|
||||||
|
path.join(tmpdir(), `voice-roundtrip-agent-${targetProvider}-`)
|
||||||
|
);
|
||||||
|
const voiceAgent = await withTimeout(
|
||||||
|
"createVoiceTargetAgent",
|
||||||
|
30000,
|
||||||
|
ctx.client.createAgent({
|
||||||
|
config: {
|
||||||
|
...getFullAccessConfig(targetProvider),
|
||||||
|
cwd: voiceCwd,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const voiceAgentId = voiceAgent.id;
|
||||||
const voiceMode = await withTimeout(
|
const voiceMode = await withTimeout(
|
||||||
"setVoiceMode",
|
"setVoiceMode",
|
||||||
15000,
|
15000,
|
||||||
@@ -260,6 +275,8 @@ describe("voice roundtrip e2e", () => {
|
|||||||
} finally {
|
} finally {
|
||||||
offStream();
|
offStream();
|
||||||
offErrors();
|
offErrors();
|
||||||
|
await ctx.client.setVoiceMode(false).catch(() => undefined);
|
||||||
|
rmSync(voiceCwd, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
|
|
||||||
@@ -281,7 +298,8 @@ describe("voice roundtrip e2e", () => {
|
|||||||
|
|
||||||
expect(normalized.length).toBeGreaterThan(0);
|
expect(normalized.length).toBeGreaterThan(0);
|
||||||
expect(normalized).toMatch(/round|trip|successful/);
|
expect(normalized).toMatch(/round|trip|successful/);
|
||||||
},
|
},
|
||||||
180000
|
180000
|
||||||
);
|
);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -83,10 +83,6 @@ export class VoiceAssistantWebSocketServer {
|
|||||||
};
|
};
|
||||||
} | null;
|
} | null;
|
||||||
private readonly voice: {
|
private readonly voice: {
|
||||||
voiceLlmProvider?: AgentProvider | null;
|
|
||||||
voiceLlmModeId?: string | null;
|
|
||||||
voiceLlmProviderExplicit?: boolean;
|
|
||||||
voiceLlmModel?: string | null;
|
|
||||||
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
|
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
|
||||||
ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
|
ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
|
||||||
removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
|
removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
|
||||||
@@ -110,10 +106,6 @@ export class VoiceAssistantWebSocketServer {
|
|||||||
speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
|
speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
|
||||||
terminalManager?: TerminalManager | null,
|
terminalManager?: TerminalManager | null,
|
||||||
voice?: {
|
voice?: {
|
||||||
voiceLlmProvider?: AgentProvider | null;
|
|
||||||
voiceLlmModeId?: string | null;
|
|
||||||
voiceLlmProviderExplicit?: boolean;
|
|
||||||
voiceLlmModel?: string | null;
|
|
||||||
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
|
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
|
||||||
ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
|
ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
|
||||||
removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
|
removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
|
||||||
|
|||||||
@@ -348,7 +348,7 @@ export const ArchiveAgentRequestMessageSchema = z.object({
|
|||||||
export const SetVoiceModeMessageSchema = z.object({
|
export const SetVoiceModeMessageSchema = z.object({
|
||||||
type: z.literal("set_voice_mode"),
|
type: z.literal("set_voice_mode"),
|
||||||
enabled: z.boolean(),
|
enabled: z.boolean(),
|
||||||
voiceAgentId: z.string().optional(),
|
agentId: z.string().optional(),
|
||||||
requestId: z.string().optional(),
|
requestId: z.string().optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -578,7 +578,7 @@ export const SetVoiceModeResponseMessageSchema = z.object({
|
|||||||
payload: z.object({
|
payload: z.object({
|
||||||
requestId: z.string(),
|
requestId: z.string(),
|
||||||
enabled: z.boolean(),
|
enabled: z.boolean(),
|
||||||
voiceAgentId: z.string().nullable(),
|
agentId: z.string().nullable(),
|
||||||
accepted: z.boolean(),
|
accepted: z.boolean(),
|
||||||
error: z.string().nullable(),
|
error: z.string().nullable(),
|
||||||
}),
|
}),
|
||||||
|
|||||||
Reference in New Issue
Block a user