jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Update files

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-07 14:06:11 +07:00
parent cef7d8c134
commit 3cc7feab21
19 changed files with 556 additions and 630 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/components/agent-input-area.tsx
+78 -2
View File
@@ -4,10 +4,11 @@ import {
Text,
ActivityIndicator,
Platform,
Alert,
} from "react-native";
import { useState, useEffect, useRef, useCallback, useMemo } from "react";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { ArrowUp, Square, Pencil } from "lucide-react-native";
import { ArrowUp, Square, Pencil, AudioLines } from "lucide-react-native";
import Animated, { useAnimatedStyle } from "react-native-reanimated";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
import { useSafeAreaInsets } from "react-native-safe-area-context";
@@ -30,6 +31,7 @@ import { useAgentCommandsQuery } from "@/hooks/use-agent-commands-query";
import { encodeImages } from "@/utils/encode-images";
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
import { focusWithRetries } from "@/utils/web-focus";
import { useVoiceOptional } from "@/contexts/voice-context";
type QueuedMessage = {
id: string;
@@ -76,6 +78,7 @@ export function AgentInputArea({
const client = useSessionStore(
(state) => state.sessions[serverId]?.client ?? null
);
const voice = useVoiceOptional();
const isConnected = client?.isConnected ?? false;
const agent = useSessionStore((state) =>
@@ -411,6 +414,29 @@ export function AgentInputArea({
messageInputRef.current?.focus();
}
const isVoiceModeForAgent =
voice?.isVoiceModeForAgent(serverId, agentId) ?? false;
const handleToggleRealtimeVoice = useCallback(() => {
if (!voice || !isConnected) {
return;
}
if (voice.isVoiceSwitching) {
return;
}
if (voice.isVoiceModeForAgent(serverId, agentId)) {
void voice.stopVoice().catch((error) => {
console.error("[AgentInputArea] Failed to stop voice mode", error);
Alert.alert("Voice failed", "Unable to stop realtime voice mode.");
});
return;
}
void voice.startVoice(serverId, agentId).catch((error) => {
console.error("[AgentInputArea] Failed to start voice mode", error);
Alert.alert("Voice failed", "Unable to start realtime voice mode.");
});
}, [agentId, isConnected, serverId, voice]);
function handleEditQueuedMessage(id: string) {
const item = queuedMessages.find((q) => q.id === id);
if (!item) return;
@@ -494,7 +520,7 @@ export function AgentInputArea({
]
);
const rightContent = isAgentRunning ? (
const cancelButton = isAgentRunning ? (
<Pressable
onPress={handleCancelAgent}
disabled={!isConnected || isCancellingAgent}
@@ -515,6 +541,35 @@ export function AgentInputArea({
</Pressable>
) : null;
const rightContent = (
<View style={styles.rightControls}>
<Pressable
onPress={handleToggleRealtimeVoice}
disabled={!isConnected || voice?.isVoiceSwitching}
accessibilityLabel={
isVoiceModeForAgent ? "Disable realtime voice mode" : "Enable realtime voice mode"
}
accessibilityRole="button"
style={[
styles.realtimeVoiceButton as any,
isVoiceModeForAgent ? (styles.realtimeVoiceButtonActive as any) : undefined,
(!isConnected || voice?.isVoiceSwitching
? styles.buttonDisabled
: undefined) as any,
]}
>
{voice?.isVoiceSwitching ? (
<ActivityIndicator size="small" color="white" />
) : isVoiceModeForAgent ? (
<Square size={16} color="white" fill="white" />
) : (
<AudioLines size={18} color={theme.colors.foreground} />
)}
</Pressable>
{cancelButton}
</View>
);
const leftContent = <AgentStatusBar agentId={agentId} serverId={serverId} />;
return (
@@ -588,6 +643,8 @@ export function AgentInputArea({
isScreenFocused={isScreenFocused}
leftContent={leftContent}
rightContent={rightContent}
voiceServerId={serverId}
voiceAgentId={agentId}
isAgentRunning={isAgentRunning}
onQueue={handleQueue}
onKeyPress={handleCommandKeyPress}
@@ -631,6 +688,25 @@ const styles = StyleSheet.create(((theme: Theme) => ({
alignItems: "center",
justifyContent: "center",
},
rightControls: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[2],
},
realtimeVoiceButton: {
width: 34,
height: 34,
borderRadius: theme.borderRadius.full,
backgroundColor: theme.colors.surface0,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
alignItems: "center",
justifyContent: "center",
},
realtimeVoiceButtonActive: {
backgroundColor: theme.colors.palette.green[600],
borderColor: theme.colors.palette.green[800],
},
buttonDisabled: {
opacity: 0.5,
},
packages/app/src/components/agent-stream-view.tsx
+2 -1
View File
@@ -102,7 +102,7 @@ export function AgentStreamView({
const flatListRef = useRef<FlatList<StreamItem>>(null);
const { theme } = useUnistyles();
const insets = useSafeAreaInsets();
const { isVoiceMode } = useVoice();
const { isVoiceModeForAgent } = useVoice();
const [isNearBottom, setIsNearBottom] = useState(true);
const hasScrolledInitially = useRef(false);
const hasAutoScrolledOnce = useRef(false);
@@ -114,6 +114,7 @@ export function AgentStreamView({
// Get serverId (fallback to agent's serverId if not provided)
const resolvedServerId = serverId ?? agent.serverId ?? "";
const isVoiceMode = isVoiceModeForAgent(resolvedServerId, agentId);
const client = useSessionStore(
(state) => state.sessions[resolvedServerId]?.client ?? null
packages/app/src/components/message-input.tsx
+12 -3
View File
@@ -61,6 +61,8 @@ export interface MessageInputProps {
leftContent?: React.ReactNode;
/** Content to render on the right side after voice button (e.g., realtime button, cancel button) */
rightContent?: React.ReactNode;
voiceServerId?: string;
voiceAgentId?: string;
/** When true and there's sendable content, calls onQueue instead of onSubmit */
isAgentRunning?: boolean;
/** Callback for queue button when agent is running */
@@ -117,6 +119,8 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
isScreenFocused = true,
leftContent,
rightContent,
voiceServerId,
voiceAgentId,
isAgentRunning = false,
onQueue,
onKeyPress: onKeyPressCallback,
@@ -310,15 +314,20 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
}));
const handleVoicePress = useCallback(async () => {
const isRealtimeVoiceForCurrentAgent =
voice &&
!!voiceServerId &&
!!voiceAgentId &&
voice.isVoiceModeForAgent(voiceServerId, voiceAgentId);
if (isDictating) {
await cancelDictation();
} else {
if (voice?.isVoiceMode) {
if (isRealtimeVoiceForCurrentAgent) {
await voice.stopVoice();
}
await startDictation();
}
}, [isDictating, cancelDictation, startDictation, voice]);
}, [isDictating, cancelDictation, startDictation, voice, voiceAgentId, voiceServerId]);
const handleCancelRecording = useCallback(async () => {
await cancelDictation();
@@ -626,6 +635,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
<Mic size={20} color={theme.colors.foreground} />
)}
</Pressable>
{rightContent}
{shouldShowSendButton && (
<Pressable
onPress={handleSendMessage}
@@ -647,7 +657,6 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
<ArrowUp size={20} color="white" />
</Pressable>
)}
{!shouldShowSendButton && rightContent}
</View>
</View>
</Animated.View>
packages/app/src/components/sliding-sidebar.tsx
+2 -198
View File
@@ -1,5 +1,5 @@
import { useCallback, useMemo, useState, useEffect } from "react";
import { View, Pressable, Text, Platform, Alert, ActivityIndicator } from "react-native";
import { View, Pressable, Text, Platform } from "react-native";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import Animated, {
useAnimatedStyle,
@@ -9,21 +9,17 @@ import Animated, {
} from "react-native-reanimated";
import { Gesture, GestureDetector } from "react-native-gesture-handler";
import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles";
import { Plus, Settings, Users, AudioLines } from "lucide-react-native";
import { Plus, Settings, Users } from "lucide-react-native";
import { router } from "expo-router";
import { usePanelStore } from "@/stores/panel-store";
import { GroupedAgentList } from "./grouped-agent-list";
import { useAggregatedAgents } from "@/hooks/use-aggregated-agents";
import { useSidebarAnimation } from "@/contexts/sidebar-animation-context";
import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window";
import { useVoice } from "@/contexts/voice-context";
import { useDaemonConnections } from "@/contexts/daemon-connections-context";
import { VoicePanel } from "./voice-panel";
import { useSidebarAgentSections } from "@/hooks/use-sidebar-agent-sections";
import { useSidebarCollapsedSectionsStore } from "@/stores/sidebar-collapsed-sections-store";
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
import { deriveSidebarShortcutAgentKeys } from "@/utils/sidebar-shortcuts";
import { AdaptiveModalSheet } from "@/components/adaptive-modal-sheet";
const DESKTOP_SIDEBAR_WIDTH = 320;
const SIDEBAR_AGENT_LIMIT = 50; // temporary for screenshots
@@ -56,9 +52,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
} = useSidebarAnimation();
const trafficLightPadding = useTrafficLightPadding();
const dragHandlers = useTauriDragHandlers();
const { connectionStates } = useDaemonConnections();
const { isVoiceMode, isVoiceSwitching, startVoice, stopVoice } = useVoice();
const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false);
// Track user-initiated refresh to avoid showing spinner on background revalidation
const [isManualRefresh, setIsManualRefresh] = useState(false);
@@ -142,65 +135,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
router.push("/agents");
}, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]);
const voiceEligibleHosts = useMemo(() => {
return Array.from(connectionStates.values()).filter((entry) => entry.status === "online");
}, [connectionStates]);
const hasAnyConfiguredHosts = connectionStates.size > 0;
const handleToggleVoice = useCallback(() => {
if (isVoiceSwitching) {
return;
}
if (isVoiceMode) {
void stopVoice().catch((error) => {
console.error("[SlidingSidebar] Failed to stop voice", error);
Alert.alert("Voice failed", "Unable to stop voice mode.");
});
return;
}
if (voiceEligibleHosts.length === 0) {
if (!hasAnyConfiguredHosts) {
Alert.alert(
"No hosts available",
"Add a host in Settings before starting Voice mode.",
[{ text: "Open Settings", onPress: () => router.push("/settings") }, { text: "OK" }]
);
return;
}
Alert.alert(
"Hosts reconnecting",
"Every host is offline right now. Paseo reconnects automatically—try Voice mode again once one comes online."
);
return;
}
if (voiceEligibleHosts.length === 1) {
void startVoice(voiceEligibleHosts[0].daemon.serverId).catch((error) => {
console.error("[SlidingSidebar] Failed to start voice", error);
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
});
return;
}
setShowVoiceHostPicker(true);
}, [hasAnyConfiguredHosts, isVoiceMode, isVoiceSwitching, startVoice, stopVoice, voiceEligibleHosts]);
const handleSelectVoiceHost = useCallback(
(serverId: string) => {
setShowVoiceHostPicker(false);
void startVoice(serverId).catch((error) => {
console.error("[SlidingSidebar] Failed to start voice", error);
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
});
},
[startVoice]
);
const handleDismissVoiceHostPicker = useCallback(() => {
setShowVoiceHostPicker(false);
}, []);
// Close gesture (swipe left to close when sidebar is open)
// Only activates on leftward swipe, fails on rightward or vertical movement
// This mirrors the explorer-sidebar pattern for the right sidebar
@@ -301,8 +235,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
parentGestureRef={closeGestureRef}
/>
{isVoiceMode ? <VoicePanel /> : null}
{/* Footer */}
<View style={styles.sidebarFooter}>
<Pressable
@@ -319,43 +251,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
)}
</Pressable>
<View style={styles.footerIconRow}>
<Pressable
style={styles.footerIconButton}
testID="sidebar-voice"
nativeID="sidebar-voice"
collapsable={false}
accessible
accessibilityLabel="Voice"
accessibilityRole="button"
disabled={isVoiceSwitching}
onPress={handleToggleVoice}
>
{({ hovered }) => (
isVoiceSwitching ? (
<ActivityIndicator
size="small"
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
) : (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)
)}
</Pressable>
<Pressable
style={styles.footerIconButton}
testID="sidebar-settings"
@@ -375,32 +270,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
</View>
</Animated.View>
</GestureDetector>
<AdaptiveModalSheet
title="Choose a host"
visible={showVoiceHostPicker}
onClose={handleDismissVoiceHostPicker}
snapPoints={["45%", "70%"]}
testID="voice-host-picker"
>
<View style={styles.hostPickerList}>
{voiceEligibleHosts.map((entry) => (
<Pressable
key={entry.daemon.serverId}
style={styles.hostPickerOption}
onPress={() => handleSelectVoiceHost(entry.daemon.serverId)}
>
<Text style={styles.hostPickerOptionText}>{entry.daemon.label}</Text>
</Pressable>
))}
<Pressable
style={styles.hostPickerCancel}
onPress={handleDismissVoiceHostPicker}
>
<Text style={styles.hostPickerCancelText}>Cancel</Text>
</Pressable>
</View>
</AdaptiveModalSheet>
</View>
);
}
@@ -439,8 +308,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
selectedAgentId={selectedAgentId}
/>
{isVoiceMode ? <VoicePanel /> : null}
{/* Footer */}
<View style={styles.sidebarFooter}>
<Pressable
@@ -457,43 +324,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
)}
</Pressable>
<View style={styles.footerIconRow}>
<Pressable
style={styles.footerIconButton}
testID="sidebar-voice"
nativeID="sidebar-voice"
collapsable={false}
accessible
accessibilityLabel="Voice"
accessibilityRole="button"
disabled={isVoiceSwitching}
onPress={handleToggleVoice}
>
{({ hovered }) => (
isVoiceSwitching ? (
<ActivityIndicator
size="small"
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
) : (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)
)}
</Pressable>
<Pressable
style={styles.footerIconButton}
testID="sidebar-settings"
@@ -510,32 +340,6 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
</Pressable>
</View>
</View>
<AdaptiveModalSheet
title="Choose a host"
visible={showVoiceHostPicker}
onClose={handleDismissVoiceHostPicker}
snapPoints={["45%", "70%"]}
testID="voice-host-picker"
>
<View style={styles.hostPickerList}>
{voiceEligibleHosts.map((entry) => (
<Pressable
key={entry.daemon.serverId}
style={styles.hostPickerOption}
onPress={() => handleSelectVoiceHost(entry.daemon.serverId)}
>
<Text style={styles.hostPickerOptionText}>{entry.daemon.label}</Text>
</Pressable>
))}
<Pressable
style={styles.hostPickerCancel}
onPress={handleDismissVoiceHostPicker}
>
<Text style={styles.hostPickerCancelText}>Cancel</Text>
</Pressable>
</View>
</AdaptiveModalSheet>
</View>
);
}
packages/app/src/contexts/voice-context.tsx
+30 -7
View File
@@ -18,10 +18,12 @@ interface VoiceContextValue {
isDetecting: boolean;
isSpeaking: boolean;
segmentDuration: number;
startVoice: (serverId: string) => Promise<void>;
startVoice: (serverId: string, agentId: string) => Promise<void>;
stopVoice: () => Promise<void>;
isVoiceModeForAgent: (serverId: string, agentId: string) => boolean;
toggleMute: () => void;
activeServerId: string | null;
activeAgentId: string | null;
}
const VoiceContext = createContext<VoiceContextValue | null>(null);
@@ -45,6 +47,7 @@ interface VoiceProviderProps {
export function VoiceProvider({ children }: VoiceProviderProps) {
const getSession = useSessionStore((state) => state.getSession);
const [activeServerId, setActiveServerId] = useState<string | null>(null);
const [activeAgentId, setActiveAgentId] = useState<string | null>(null);
const activeSession = useSessionStore(
useCallback(
(state: ReturnType<typeof useSessionStore.getState>) => {
@@ -151,7 +154,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
voiceTransportReadyRef.current = false;
}
if (!isVoiceMode || !activeServerId || !client) {
if (!isVoiceMode || !activeServerId || !activeAgentId || !client) {
wasVoiceSocketConnectedRef.current = connected;
if (!isVoiceMode) {
lastVoiceModeSyncedClientRef.current = null;
@@ -167,7 +170,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
voiceResyncInFlightRef.current = true;
voiceTransportReadyRef.current = false;
setIsVoiceSwitching(true);
void client.setVoiceMode(true).then(
void client.setVoiceMode(true, activeAgentId).then(
() => {
console.log("[Voice] Re-synced voice mode after reconnect");
lastVoiceModeSyncedClientRef.current = client;
@@ -184,7 +187,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
}
wasVoiceSocketConnectedRef.current = connected;
}, [activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]);
}, [activeAgentId, activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]);
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
@@ -201,7 +204,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
}, [isPlayingAudio]);
const startVoice = useCallback(
async (serverId: string) => {
async (serverId: string, agentId: string) => {
const session = getSession(serverId) ?? null;
if (!session) {
throw new Error(`Host ${serverId} is not connected`);
@@ -210,13 +213,23 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
setIsVoiceSwitching(true);
voiceTransportReadyRef.current = false;
try {
const previousSession = realtimeSessionRef.current;
if (
isVoiceMode &&
previousSession?.client &&
(activeServerId !== serverId || activeAgentId !== agentId)
) {
await previousSession.client.setVoiceMode(false);
}
realtimeSessionRef.current = session;
setActiveServerId(serverId);
setActiveAgentId(agentId);
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
console.warn("[Voice] Failed to activate keep-awake:", error);
});
if (session?.client) {
await session.client.setVoiceMode(true);
await session.client.setVoiceMode(true, agentId);
} else {
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
@@ -230,13 +243,14 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
console.error("[Voice] Failed to start:", error);
await realtimeAudio.stop().catch(() => undefined);
setActiveServerId((current) => (current === serverId ? null : current));
setActiveAgentId((current) => (current === agentId ? null : current));
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
throw error;
} finally {
setIsVoiceSwitching(false);
}
},
[getSession, realtimeAudio]
[activeAgentId, activeServerId, getSession, isVoiceMode, realtimeAudio]
);
const stopVoice = useCallback(async () => {
@@ -254,6 +268,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
await realtimeAudio.stop();
setIsVoiceMode(false);
setActiveServerId(null);
setActiveAgentId(null);
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
console.log("[Voice] Mode disabled");
} catch (error: any) {
@@ -265,6 +280,12 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
}
}, [realtimeAudio]);
const isVoiceModeForAgent = useCallback(
(serverId: string, agentId: string) =>
isVoiceMode && activeServerId === serverId && activeAgentId === agentId,
[activeAgentId, activeServerId, isVoiceMode]
);
const value: VoiceContextValue = {
isVoiceMode,
isVoiceSwitching,
@@ -275,8 +296,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
segmentDuration: realtimeAudio.segmentDuration,
startVoice,
stopVoice,
isVoiceModeForAgent,
toggleMute: realtimeAudio.toggleMute,
activeServerId,
activeAgentId,
};
return (
packages/server/scripts/voice-roundtrip-debug.ts
+16 -2
View File
@@ -1,4 +1,6 @@
import { randomUUID } from "node:crypto";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import pino from "pino";
import { createTestPaseoDaemon } from "../src/server/test-utils/paseo-daemon.js";
@@ -48,9 +50,20 @@ async function main(): Promise<void> {
await client.connect();
client.subscribeAgentUpdates({ subscriptionId: "voice-debug" });
const voiceAgentId = randomUUID();
const voiceCwd = mkdtempSync(path.join(tmpdir(), "voice-roundtrip-debug-"));
const voiceAgent = await client.createAgent({
config: {
provider: "claude",
cwd: voiceCwd,
modeId: "bypassPermissions",
},
});
const voiceAgentId = voiceAgent.id;
const mode = await client.setVoiceMode(true, voiceAgentId);
console.log("set_voice_mode_response", mode);
if (!mode.accepted) {
throw new Error(`setVoiceMode rejected: ${mode.error ?? "unknown error"}`);
}
const offTranscript = client.on("transcription_result", (msg) => {
if (msg.type !== "transcription_result") return;
@@ -149,6 +162,7 @@ async function main(): Promise<void> {
console.log("success", { audioChunkCount });
await client.setVoiceMode(false);
rmSync(voiceCwd, { recursive: true, force: true });
offTranscript();
offActivity();
packages/server/src/client/daemon-client.ts
+2 -2
View File
@@ -1291,13 +1291,13 @@ export class DaemonClient {
async setVoiceMode(
enabled: boolean,
voiceAgentId?: string
agentId?: string
): Promise<SetVoiceModePayload> {
const requestId = this.createRequestId();
const message = SessionInboundMessageSchema.parse({
type: "set_voice_mode",
enabled,
...(voiceAgentId ? { voiceAgentId } : {}),
...(agentId ? { agentId } : {}),
requestId,
});
return this.sendRequest({
packages/server/src/server/agent/agent-manager.ts
+20 -13
View File
@@ -407,22 +407,24 @@ export class AgentManager {
);
}
async refreshAgentFromPersistence(agentId: string): Promise<ManagedAgent> {
async refreshAgentFromPersistence(
agentId: string,
overrides?: Partial<AgentSessionConfig>
): Promise<ManagedAgent> {
const existing = this.requireAgent(agentId);
const handle = existing.persistence;
if (!handle) {
throw new Error(
`Agent ${agentId} cannot be refreshed because it has no persistence handle`
);
}
const client = this.requireClient(handle.provider);
const overrides = {
const provider = handle?.provider ?? existing.provider;
const client = this.requireClient(provider);
const refreshConfig = {
...existing.config,
provider: handle.provider,
};
...overrides,
provider,
} as AgentSessionConfig;
const normalizedConfig = await this.normalizeConfig(refreshConfig);
const session = await client.resumeSession(handle, overrides);
const session = handle
? await client.resumeSession(handle, normalizedConfig)
: await client.createSession(normalizedConfig);
// Remove the existing agent entry before swapping sessions
this.agents.delete(agentId);
@@ -436,7 +438,12 @@ export class AgentManager {
}
// Preserve existing labels during refresh
return this.registerSession(session, overrides, agentId, { labels: existing.labels });
return this.registerSession(session, normalizedConfig, agentId, {
labels: existing.labels,
createdAt: existing.createdAt,
updatedAt: existing.updatedAt,
lastUserMessageAt: existing.lastUserMessageAt,
});
}
async closeAgent(agentId: string): Promise<void> {
packages/server/src/server/agent/mcp-server.ts
+6 -1
View File
@@ -57,6 +57,7 @@ export interface AgentMcpServerOptions {
callerAgentId: string
) => VoiceCallerContext | null;
enableVoiceTools?: boolean;
voiceOnly?: boolean;
logger: Logger;
}
@@ -393,7 +394,7 @@ export async function createAgentMcpServer(
initialMode: topLevelInputSchema.initialMode.optional(),
});
if (options.enableVoiceTools || callerContext?.enableVoiceTools) {
if (options.voiceOnly || options.enableVoiceTools || callerContext?.enableVoiceTools) {
server.registerTool(
"speak",
{
@@ -432,6 +433,10 @@ export async function createAgentMcpServer(
);
}
if (options.voiceOnly) {
return server;
}
server.registerTool(
"create_agent",
{
packages/server/src/server/bootstrap.ts
+2 -97
View File
@@ -64,7 +64,6 @@ import type {
AgentClient,
AgentProvider,
} from "./agent/agent-sdk-types.js";
import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js";
import { acquirePidLock, releasePidLock } from "./pid-lock.js";
import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js";
import {
@@ -278,98 +277,8 @@ export async function createPaseoDaemon(
logger.info(
`Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`
);
const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null;
const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false;
const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS
.filter((definition) => definition.voice?.enabled)
.map((definition) => definition.id as AgentProvider);
logger.info(
{
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
voiceEnabledProviders,
},
"Voice LLM provider reconciliation started"
);
const providerClients = createAllClients(logger);
Object.assign(providerClients, config.agentClients);
const voiceLlmAvailability = Object.fromEntries(
voiceEnabledProviders.map((provider) => [provider, false])
) as Record<AgentProvider, boolean>;
for (const provider of voiceEnabledProviders) {
try {
voiceLlmAvailability[provider] = await providerClients[provider].isAvailable();
} catch (error) {
logger.warn({ err: error, provider }, "Voice LLM provider availability check failed");
voiceLlmAvailability[provider] = false;
}
}
let resolvedVoiceLlmProvider: AgentProvider | null = null;
if (requestedVoiceLlmProvider) {
if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) {
logger.error(
{ provider: requestedVoiceLlmProvider, voiceEnabledProviders },
"Configured voice LLM provider does not support voice mode"
);
throw new Error(
`Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode`
);
}
if (!voiceLlmAvailability[requestedVoiceLlmProvider]) {
logger.error(
{ provider: requestedVoiceLlmProvider, voiceLlmAvailability },
"Configured voice LLM provider is unavailable"
);
throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`);
}
resolvedVoiceLlmProvider = requestedVoiceLlmProvider;
} else {
resolvedVoiceLlmProvider =
voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null;
}
let resolvedVoiceLlmModeId: string | null = null;
let resolvedVoiceLlmModel: string | null = null;
if (!resolvedVoiceLlmProvider) {
if (voiceLlmProviderExplicit) {
logger.error(
{ requestedVoiceLlmProvider, voiceLlmAvailability },
"No voice LLM provider available"
);
throw new Error("No voice LLM provider available");
}
logger.warn(
{ requestedVoiceLlmProvider, voiceLlmAvailability },
"No default voice LLM provider available; voice mode will be disabled until a provider is configured"
);
} else {
const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
(definition) => definition.id === resolvedVoiceLlmProvider
);
if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
throw new Error(
`Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
);
}
resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
resolvedVoiceLlmModel =
config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;
}
logger.info(
{
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
resolvedVoiceLlmProvider,
resolvedVoiceLlmModeId,
resolvedVoiceLlmModel,
voiceLlmAvailability,
},
"Voice LLM provider reconciliation completed"
"Voice mode configured for agent-scoped resume flow (no dedicated voice assistant provider)"
);
let wsServer: VoiceAssistantWebSocketServer | null = null;
let voiceMcpBridgeManager: VoiceMcpSocketBridgeManager | null = null;
@@ -523,7 +432,7 @@ export async function createPaseoDaemon(
agentStorage,
paseoHome: config.paseoHome,
callerAgentId,
enableVoiceTools: false,
voiceOnly: true,
resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
logger,
@@ -555,10 +464,6 @@ export async function createPaseoDaemon(
{ stt: sttService, tts: ttsService },
terminalManager,
{
voiceLlmProvider: resolvedVoiceLlmProvider,
voiceLlmModeId: resolvedVoiceLlmModeId,
voiceLlmProviderExplicit,
voiceLlmModel: resolvedVoiceLlmModel,
voiceAgentMcpStdio: {
command: voiceMcpBridgeCommand.command,
baseArgs: [...voiceMcpBridgeCommand.baseArgs],
packages/server/src/server/daemon-client.e2e.test.ts
+19 -5
View File
@@ -142,23 +142,29 @@ describe("daemon client E2E", () => {
const agents = await ctx.client.fetchAgents();
expect(Array.isArray(agents)).toBe(true);
const voiceAgents = await ctx.client.fetchAgents({
filter: { labels: { surface: "voice" } },
const cwd = tmpCwd();
const created = await ctx.client.createAgent({
config: {
...getFullAccessConfig("codex"),
cwd,
},
});
expect(Array.isArray(voiceAgents)).toBe(true);
await expect(ctx.client.setVoiceMode(true)).resolves.toMatchObject({
await expect(ctx.client.setVoiceMode(true, created.id)).resolves.toMatchObject({
enabled: true,
agentId: created.id,
accepted: true,
error: null,
});
await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({
enabled: false,
agentId: null,
accepted: true,
error: null,
});
await ctx.client.deleteAgent(randomUUID());
rmSync(cwd, { recursive: true, force: true });
}, 30000);
test("emits server_info on websocket connect", async () => {
@@ -655,7 +661,14 @@ describe("daemon client E2E", () => {
speechTest(
"voice mode buffers audio until isLast and emits transcription_result",
async () => {
await ctx.client.setVoiceMode(true, randomUUID());
const voiceCwd = tmpCwd();
const voiceAgent = await ctx.client.createAgent({
config: {
...getFullAccessConfig("codex"),
cwd: voiceCwd,
},
});
await ctx.client.setVoiceMode(true, voiceAgent.id);
const transcription = waitForSignal(30_000, (resolve) => {
const unsubscribe = ctx.client.on("transcription_result", (message) => {
@@ -751,6 +764,7 @@ describe("daemon client E2E", () => {
} finally {
await Promise.allSettled([transcription, errorSignal]);
await ctx.client.setVoiceMode(false);
rmSync(voiceCwd, { recursive: true, force: true });
}
},
90_000
packages/server/src/server/session.ts
+226 -210
View File
@@ -1,5 +1,5 @@
import { v4 as uuidv4 } from "uuid";
import { mkdir, stat } from "fs/promises";
import { stat } from "fs/promises";
import { exec } from "child_process";
import { promisify } from "util";
import { join, resolve, sep } from "path";
@@ -58,6 +58,7 @@ import type {
AgentPermissionResponse,
AgentPromptContentBlock,
AgentPromptInput,
McpServerConfig,
AgentSessionConfig,
AgentStreamEvent,
AgentProvider,
@@ -191,7 +192,18 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
);
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
const VoiceAgentIdSchema = z.string().uuid();
const AgentIdSchema = z.string().uuid();
const VOICE_MCP_SERVER_NAME = "paseo_voice";
const VOICE_MODE_ENFORCED_MODE_ID: Partial<Record<AgentProvider, string>> = {
claude: "default",
codex: "read-only",
};
type VoiceModeBaseConfig = {
systemPrompt?: string;
mcpServers?: Record<string, McpServerConfig>;
modeId?: string;
};
interface AudioBufferState {
chunks: Buffer[];
@@ -214,10 +226,6 @@ export type SessionOptions = {
tts: TextToSpeechProvider | null;
terminalManager: TerminalManager | null;
voice?: {
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
};
voiceBridge?: {
@@ -376,10 +384,6 @@ export class Session {
} | null = null;
private readonly terminalManager: TerminalManager | null;
private terminalSubscriptions: Map<string, () => void> = new Map();
private readonly voiceLlmProvider: AgentProvider | null;
private readonly voiceLlmModeId: string | null;
private readonly voiceLlmProviderExplicit: boolean;
private readonly voiceLlmModel: string | null;
private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null;
private readonly localSpeechModelsDir: string;
private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[];
@@ -395,7 +399,8 @@ export class Session {
private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
private readonly ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
private readonly removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
private voiceAssistantAgentId: string | null = null;
private voiceModeAgentId: string | null = null;
private voiceModeBaseConfig: VoiceModeBaseConfig | null = null;
constructor(options: SessionOptions) {
const {
@@ -425,10 +430,6 @@ export class Session {
this.agentStorage = agentStorage;
this.createAgentMcpTransport = createAgentMcpTransport;
this.terminalManager = terminalManager;
this.voiceLlmProvider = voice?.voiceLlmProvider ?? null;
this.voiceLlmModeId = voice?.voiceLlmModeId ?? null;
this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false;
this.voiceLlmModel = voice?.voiceLlmModel ?? null;
this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null;
const configuredModelsDir = dictation?.localModels?.modelsDir?.trim();
this.localSpeechModelsDir =
@@ -896,7 +897,7 @@ export class Session {
break;
case "set_voice_mode":
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId, msg.requestId);
await this.handleSetVoiceMode(msg.enabled, msg.agentId, msg.requestId);
break;
case "send_agent_message_request":
@@ -1276,108 +1277,216 @@ export class Session {
*/
private async handleSetVoiceMode(
enabled: boolean,
voiceAgentId?: string,
agentId?: string,
requestId?: string
): Promise<void> {
if (enabled) {
let normalizedVoiceAgentId: string | null = null;
if (voiceAgentId && voiceAgentId.trim().length > 0) {
normalizedVoiceAgentId = this.parseVoiceAgentId(
voiceAgentId,
try {
if (enabled) {
const normalizedAgentId = this.parseVoiceTargetAgentId(
agentId ?? "",
"set_voice_mode"
);
} else {
normalizedVoiceAgentId = await this.findExistingVoiceAssistantAgentId();
if (
this.isVoiceMode &&
this.voiceModeAgentId &&
this.voiceModeAgentId !== normalizedAgentId
) {
await this.disableVoiceModeForActiveAgent(true);
}
if (!this.isVoiceMode || this.voiceModeAgentId !== normalizedAgentId) {
const refreshedAgentId = await this.enableVoiceModeForAgent(normalizedAgentId);
this.voiceModeAgentId = refreshedAgentId;
}
this.isVoiceMode = true;
this.sessionLogger.info(
{
agentId: this.voiceModeAgentId,
},
"Voice mode enabled for existing agent"
);
if (requestId) {
this.emit({
type: "set_voice_mode_response",
payload: {
requestId,
enabled: true,
agentId: this.voiceModeAgentId,
accepted: true,
error: null,
},
});
}
return;
}
this.isVoiceMode = true;
this.voiceAssistantAgentId = normalizedVoiceAgentId;
this.sessionLogger.info(
await this.disableVoiceModeForActiveAgent(true);
this.isVoiceMode = false;
this.sessionLogger.info("Voice mode disabled");
if (requestId) {
this.emit({
type: "set_voice_mode_response",
payload: {
requestId,
enabled: false,
agentId: null,
accepted: true,
error: null,
},
});
}
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to set voice mode";
this.sessionLogger.error(
{
voiceAssistantAgentId: this.voiceAssistantAgentId,
resumedVoiceAgent: Boolean(normalizedVoiceAgentId),
providedVoiceAgentId: Boolean(voiceAgentId?.trim()),
err: error,
enabled,
requestedAgentId: agentId ?? null,
},
"Voice mode enabled (agent-backed)"
"set_voice_mode failed"
);
if (requestId) {
this.emit({
type: "set_voice_mode_response",
payload: {
requestId,
enabled: true,
voiceAgentId: this.voiceAssistantAgentId,
accepted: true,
error: null,
enabled: this.isVoiceMode,
agentId: this.voiceModeAgentId,
accepted: false,
error: errorMessage,
},
});
return;
}
return;
}
this.isVoiceMode = false;
const disabledVoiceAgentId = this.voiceAssistantAgentId;
if (disabledVoiceAgentId && this.removeVoiceMcpSocketForAgent) {
try {
await this.removeVoiceMcpSocketForAgent(disabledVoiceAgentId);
} catch (error) {
this.sessionLogger.warn(
{ err: error, voiceAssistantAgentId: disabledVoiceAgentId },
"Failed to remove voice MCP socket bridge on disable"
);
}
}
this.sessionLogger.info(
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
"Voice mode disabled (agent-backed)"
);
if (requestId) {
this.emit({
type: "set_voice_mode_response",
payload: {
requestId,
enabled: false,
voiceAgentId: this.voiceAssistantAgentId,
accepted: true,
error: null,
},
});
throw error;
}
}
private parseVoiceAgentId(rawId: string, source: string): string {
const parsed = VoiceAgentIdSchema.safeParse(rawId.trim());
private parseVoiceTargetAgentId(rawId: string, source: string): string {
const parsed = AgentIdSchema.safeParse(rawId.trim());
if (!parsed.success) {
throw new Error(`${source}: voiceAgentId must be a UUID`);
throw new Error(`${source}: agentId must be a UUID`);
}
return parsed.data;
}
private async findExistingVoiceAssistantAgentId(): Promise<string | null> {
const scopedCandidates = await this.listAgentPayloads({
labels: {
surface: "voice",
voiceClientId: this.clientId,
},
});
const fallbackCandidates = await this.listAgentPayloads({
labels: { surface: "voice" },
});
const available = [
...scopedCandidates,
...fallbackCandidates.filter(
(candidate) => !scopedCandidates.some((scoped) => scoped.id === candidate.id)
),
].filter((agent) => !agent.archivedAt && agent.labels?.ui !== "false");
if (available.length === 0) {
return null;
private cloneMcpServers(
servers: Record<string, McpServerConfig> | undefined
): Record<string, McpServerConfig> | undefined {
if (!servers) {
return undefined;
}
available.sort((left, right) => {
const leftTime = Date.parse(left.updatedAt);
const rightTime = Date.parse(right.updatedAt);
return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0);
return JSON.parse(JSON.stringify(servers)) as Record<string, McpServerConfig>;
}
private buildVoiceModeSystemPrompt(existing?: string): string {
const chunks = [existing?.trim(), VOICE_AGENT_SYSTEM_INSTRUCTION].filter(
(value): value is string => Boolean(value && value.length > 0)
);
return chunks.join("\n\n");
}
private buildVoiceModeMcpServers(
existing: Record<string, McpServerConfig> | undefined,
socketPath: string
): Record<string, McpServerConfig> {
const mcpStdio = this.voiceAgentMcpStdio;
if (!mcpStdio) {
throw new Error("Voice MCP stdio bridge is not configured");
}
return {
...(existing ?? {}),
[VOICE_MCP_SERVER_NAME]: buildVoiceAgentMcpServerConfig({
command: mcpStdio.command,
baseArgs: mcpStdio.baseArgs,
socketPath,
env: mcpStdio.env,
}),
};
}
private getVoiceModeEnforcedModeId(provider: AgentProvider): string | undefined {
return VOICE_MODE_ENFORCED_MODE_ID[provider];
}
private async enableVoiceModeForAgent(agentId: string): Promise<string> {
const ensureVoiceSocket = this.ensureVoiceMcpSocketForAgent;
if (!ensureVoiceSocket) {
throw new Error("Voice MCP socket bridge is not configured");
}
const existing = await this.ensureAgentLoaded(agentId);
const socketPath = await ensureVoiceSocket(agentId);
this.registerVoiceBridgeForAgent(agentId);
const baseConfig: VoiceModeBaseConfig = {
systemPrompt: existing.config.systemPrompt,
mcpServers: this.cloneMcpServers(existing.config.mcpServers),
modeId: existing.config.modeId,
};
this.voiceModeBaseConfig = baseConfig;
const voiceModeId = this.getVoiceModeEnforcedModeId(existing.config.provider);
const refreshOverrides: Partial<AgentSessionConfig> = {
systemPrompt: this.buildVoiceModeSystemPrompt(baseConfig.systemPrompt),
mcpServers: this.buildVoiceModeMcpServers(baseConfig.mcpServers, socketPath),
...(voiceModeId ? { modeId: voiceModeId } : {}),
};
try {
const refreshed = await this.agentManager.refreshAgentFromPersistence(
agentId,
refreshOverrides
);
return refreshed.id;
} catch (error) {
this.unregisterVoiceSpeakHandler?.(agentId);
this.unregisterVoiceCallerContext?.(agentId);
await this.removeVoiceMcpSocketForAgent?.(agentId).catch(() => undefined);
this.voiceModeBaseConfig = null;
throw error;
}
}
private async disableVoiceModeForActiveAgent(
restoreAgentConfig: boolean
): Promise<void> {
const agentId = this.voiceModeAgentId;
if (!agentId) {
this.voiceModeBaseConfig = null;
return;
}
this.unregisterVoiceSpeakHandler?.(agentId);
this.unregisterVoiceCallerContext?.(agentId);
await this.removeVoiceMcpSocketForAgent?.(agentId).catch((error) => {
this.sessionLogger.warn(
{ err: error, agentId },
"Failed to remove voice MCP socket bridge on disable"
);
});
return available[0]?.id ?? null;
if (restoreAgentConfig && this.voiceModeBaseConfig) {
const baseConfig = this.voiceModeBaseConfig;
try {
await this.agentManager.refreshAgentFromPersistence(agentId, {
systemPrompt: baseConfig.systemPrompt,
mcpServers: this.cloneMcpServers(baseConfig.mcpServers),
modeId: baseConfig.modeId,
});
} catch (error) {
this.sessionLogger.warn(
{ err: error, agentId },
"Failed to restore agent config while disabling voice mode"
);
}
}
this.voiceModeBaseConfig = null;
this.voiceModeAgentId = null;
}
/**
@@ -4501,101 +4610,11 @@ export class Session {
}
}
private async ensureVoiceAssistantAgent(): Promise<string> {
const ensureVoiceSocket = this.ensureVoiceMcpSocketForAgent;
if (!ensureVoiceSocket) {
throw new Error("Voice MCP socket bridge is not configured");
}
if (this.voiceAssistantAgentId) {
const existing = this.agentManager.getAgent(this.voiceAssistantAgentId);
if (existing) {
await ensureVoiceSocket(existing.id);
this.registerVoiceBridgeForAgent(existing.id);
return existing.id;
}
try {
const hydrated = await this.ensureAgentLoaded(this.voiceAssistantAgentId);
await ensureVoiceSocket(hydrated.id);
this.voiceAssistantAgentId = hydrated.id;
this.registerVoiceBridgeForAgent(hydrated.id);
return hydrated.id;
} catch (error) {
this.sessionLogger.debug(
{ err: error, voiceAssistantAgentId: this.voiceAssistantAgentId },
"Voice assistant agent not found in active/persisted state; creating new session"
);
}
}
const provider = this.voiceLlmProvider;
if (!provider) {
throw new Error("Voice LLM provider is not configured");
}
const voiceAgentId = this.voiceAssistantAgentId ?? uuidv4();
const cwd = join(this.paseoHome, "voice-agent-workspace");
await mkdir(cwd, { recursive: true });
const mcpStdio = this.voiceAgentMcpStdio;
if (!mcpStdio) {
throw new Error("Voice MCP stdio bridge is not configured");
}
const socketPath = await ensureVoiceSocket(voiceAgentId);
const model = this.voiceLlmModel?.trim() || undefined;
const config: AgentSessionConfig = {
provider,
cwd,
systemPrompt: VOICE_AGENT_SYSTEM_INSTRUCTION,
modeId: this.voiceLlmModeId ?? "default",
...(model ? { model } : {}),
mcpServers: {
paseo: buildVoiceAgentMcpServerConfig({
command: mcpStdio.command,
baseArgs: mcpStdio.baseArgs,
socketPath,
env: mcpStdio.env,
}),
},
};
const created = await this.agentManager.createAgent(config, voiceAgentId, {
labels: {
surface: "voice",
ui: "true",
voiceClientId: this.clientId,
},
});
this.voiceAssistantAgentId = created.id;
try {
this.registerVoiceBridgeForAgent(created.id);
} catch (error) {
this.unregisterVoiceSpeakHandler?.(created.id);
this.unregisterVoiceCallerContext?.(created.id);
await this.removeVoiceMcpSocketForAgent?.(created.id).catch(() => undefined);
await this.agentManager.closeAgent(created.id).catch(() => undefined);
this.voiceAssistantAgentId = null;
throw error;
}
this.sessionLogger.info(
{
voiceAssistantAgentId: created.id,
provider,
model: model ?? null,
providerExplicit: this.voiceLlmProviderExplicit,
},
"Voice assistant agent initialized"
);
return created.id;
}
private registerVoiceBridgeForAgent(agentId: string): void {
this.registerVoiceSpeakHandler?.(agentId, async ({ text, signal }) => {
this.sessionLogger.info(
{
voiceAssistantAgentId: agentId,
agentId,
textLength: text.length,
preview: text.slice(0, 160),
},
@@ -4609,7 +4628,7 @@ export class Session {
true
);
this.sessionLogger.info(
{ voiceAssistantAgentId: agentId, textLength: text.length },
{ agentId, textLength: text.length },
"Voice speak tool call finished playback"
);
this.emit({
@@ -4625,7 +4644,7 @@ export class Session {
this.registerVoiceCallerContext?.(agentId, {
childAgentDefaultLabels: { ui: "true" },
allowCustomCwd: true,
allowCustomCwd: false,
enableVoiceTools: true,
});
}
@@ -4643,14 +4662,16 @@ export class Session {
return isVoicePermissionAllowed(request);
}
private async processWithVoiceAgent(userText: string): Promise<void> {
const agentId = await this.ensureVoiceAssistantAgent();
private async processWithVoiceAgent(
agentId: string,
userText: string
): Promise<void> {
await this.ensureAgentLoaded(agentId);
await this.interruptAgentIfRunning(agentId);
this.sessionLogger.info(
{
voiceAssistantAgentId: agentId,
agentId,
userTextLength: userText.length,
userTextPreview: userText.slice(0, 160),
},
@@ -4670,7 +4691,7 @@ export class Session {
if (event.item.type === "tool_call" && typeof event.item.name === "string") {
this.sessionLogger.info(
{
voiceAssistantAgentId: agentId,
agentId,
toolName: event.item.name,
callId: event.item.callId,
},
@@ -4689,11 +4710,12 @@ export class Session {
} else {
await this.agentManager.respondToPermission(agentId, event.request.id, {
behavior: "deny",
message: "Voice assistant policy only allows MCP paseo tools.",
message: "Voice assistant policy only allows the speak tool.",
interrupt: true,
});
throw new Error(
`Voice assistant denied non-MCP tool request: ${event.request.name}`
this.sessionLogger.warn(
{ agentId, requestName: event.request.name, requestId: event.request.id },
"Voice assistant denied non-speak permission request"
);
}
}
@@ -4701,12 +4723,12 @@ export class Session {
if (!sawSpeakToolCall) {
this.sessionLogger.warn(
{ voiceAssistantAgentId: agentId },
{ agentId },
"Voice agent turn completed without speak tool call; no audio playback emitted"
);
} else {
this.sessionLogger.info(
{ voiceAssistantAgentId: agentId },
{ agentId },
"Voice agent turn included speak tool call"
);
}
@@ -4721,11 +4743,18 @@ export class Session {
this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled");
return;
}
const agentId = this.voiceModeAgentId;
if (!agentId) {
this.sessionLogger.warn(
"Ignoring processVoiceTurn call because no agent is currently voice-enabled"
);
return;
}
const normalized = (latestUserText ?? "").trim();
if (!normalized) {
return;
}
await this.processWithVoiceAgent(normalized);
await this.processWithVoiceAgent(agentId, normalized);
} finally {
this.turnIndex++;
this.currentStreamPromise = null;
@@ -4980,21 +5009,8 @@ export class Session {
this.agentTools = null;
}
if (this.voiceAssistantAgentId) {
try {
await this.agentManager.closeAgent(this.voiceAssistantAgentId);
} catch (error) {
this.sessionLogger.warn(
{ err: error, voiceAssistantAgentId: this.voiceAssistantAgentId },
"Failed to close voice assistant agent"
);
} finally {
this.unregisterVoiceSpeakHandler?.(this.voiceAssistantAgentId);
this.unregisterVoiceCallerContext?.(this.voiceAssistantAgentId);
await this.removeVoiceMcpSocketForAgent?.(this.voiceAssistantAgentId).catch(() => undefined);
this.voiceAssistantAgentId = null;
}
}
await this.disableVoiceModeForActiveAgent(true);
this.isVoiceMode = false;
// Unsubscribe from all terminals
for (const unsubscribe of this.terminalSubscriptions.values()) {
packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts
+13 -3
View File
@@ -1,8 +1,7 @@
import { describe, expect, test } from "vitest";
import { mkdtempSync } from "node:fs";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { randomUUID } from "node:crypto";
import pino from "pino";
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
@@ -191,7 +190,17 @@ describe("speech models (download E2E)", () => {
};
});
await ctx.client.setVoiceMode(true, randomUUID());
const voiceCwd = mkdtempSync(path.join(tmpdir(), "speech-download-voice-agent-"));
const voiceAgent = await ctx.client.createAgent({
config: {
provider: "codex",
cwd: voiceCwd,
modeId: "full-access",
model: "gpt-5.1-codex-mini",
thinkingOptionId: "low",
},
});
await ctx.client.setVoiceMode(true, voiceAgent.id);
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
const isLast = offset + chunkBytes >= pcm16.length;
@@ -202,6 +211,7 @@ describe("speech models (download E2E)", () => {
expect(voiceText).toContain("voice note");
}
await ctx.client.setVoiceMode(false);
rmSync(voiceCwd, { recursive: true, force: true });
// Streaming TTS: generate locally from downloaded model and validate chunking.
const ttsText = "This is a voice note.";
packages/server/src/server/voice-local-agent.e2e.test.ts
+89 -53
View File
@@ -1,9 +1,12 @@
import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { randomUUID } from "node:crypto";
import path from "node:path";
import { readFile } from "node:fs/promises";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import pino from "pino";
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
import { getFullAccessConfig } from "./daemon-e2e/agent-configs.js";
import { OpenAITTS } from "./speech/providers/openai/tts.js";
const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
const shouldRun =
@@ -69,61 +72,94 @@ function waitForSignal<T>(
test(
"routes voice turns through local agent speak tool",
async () => {
await ctx.client.setVoiceMode(true, randomUUID());
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
const offAudio = ctx.client.on("audio_output", (msg) => {
if (msg.type !== "audio_output") return;
resolve({ chunkId: msg.payload.id });
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offAudio();
offError();
};
});
const assistantLogPromise = waitForSignal<string>(120000, (resolve, reject) => {
const offLog = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "assistant") return;
const content = String(msg.payload.content ?? "");
if (!content.trim()) return;
resolve(content);
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offLog();
offError();
};
});
const fixturePath = path.resolve(
process.cwd(),
"..",
"app",
"e2e",
"fixtures",
"recording.wav"
const logger = pino({ level: "silent" });
const ttsProvider = new OpenAITTS(
{
apiKey: openaiApiKey!,
responseFormat: "pcm",
voice: "alloy",
},
logger
);
const wav = await readFile(fixturePath);
await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true);
const [{ chunkId }, assistantText] = await Promise.all([
audioPromise,
assistantLogPromise,
const voiceCwd = mkdtempSync(path.join(tmpdir(), "voice-local-agent-"));
const targetAgent = await ctx.client.createAgent({
config: {
...getFullAccessConfig("codex"),
cwd: voiceCwd,
},
});
const voiceMode = await ctx.client.setVoiceMode(true, targetAgent.id);
expect(voiceMode.accepted).toBe(true);
const transcriptionPromise = waitForSignal<string>(120000, (resolve, reject) => {
const offTranscript = ctx.client.on("transcription_result", (msg) => {
if (msg.type !== "transcription_result") return;
const text = String(msg.payload.text ?? "").trim();
if (!text) return;
resolve(text);
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offTranscript();
offError();
};
});
const speakToolPromise = waitForSignal<string>(120000, (resolve, reject) => {
const offStream = ctx.client.on("agent_stream", (msg) => {
if (msg.type !== "agent_stream") return;
if (msg.payload.event.type !== "timeline") return;
const item = msg.payload.event.item;
if (item.type !== "tool_call") return;
const name = String(item.name ?? "");
if (!name.toLowerCase().includes("speak")) return;
resolve(name);
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offStream();
offError();
};
});
const inputSpeech = await ttsProvider.synthesizeSpeech(
"Use the speak tool and say exactly local agent check."
);
const buffers: Buffer[] = [];
for await (const chunk of inputSpeech.stream as AsyncIterable<unknown>) {
buffers.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as ArrayBufferLike));
}
const pcm = Buffer.concat(buffers);
const chunkBytes = 4800;
for (let offset = 0; offset < pcm.length; offset += chunkBytes) {
const chunk = pcm.subarray(offset, Math.min(pcm.length, offset + chunkBytes));
const isLast = offset + chunkBytes >= pcm.length;
await ctx.client.sendVoiceAudioChunk(
chunk.toString("base64"),
"audio/pcm;rate=24000;bits=16",
isLast
);
}
const [transcript, speakToolName] = await Promise.all([
transcriptionPromise,
speakToolPromise,
]);
expect(chunkId.length).toBeGreaterThan(0);
expect(assistantText.trim().length).toBeGreaterThan(0);
await ctx.client.setVoiceMode(false).catch(() => undefined);
rmSync(voiceCwd, { recursive: true, force: true });
expect(transcript.length).toBeGreaterThan(0);
expect(speakToolName.toLowerCase()).toContain("speak");
const agents = await ctx.client.fetchAgents();
expect(
packages/server/src/server/voice-permission-policy.test.ts
+5 -5
View File
@@ -21,13 +21,13 @@ describe("isVoicePermissionAllowed", () => {
expect(result).toBe(true);
});
test("allows explicit MCP/paseo tool names", () => {
test("denies explicit MCP/paseo tool names", () => {
expect(
isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))
).toBe(true);
).toBe(false);
expect(
isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))
).toBe(true);
).toBe(false);
});
test("denies non-tool permission kinds", () => {
@@ -37,12 +37,12 @@ describe("isVoicePermissionAllowed", () => {
expect(result).toBe(false);
});
test("allows codextool only when metadata references MCP/paseo", () => {
test("allows codextool only when metadata references speak", () => {
const allowed = isVoicePermissionAllowed(
buildRequest({
name: "codextool",
metadata: {
questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }],
questions: [{ question: "Allow codextool to call speak for user feedback?" }],
},
})
);
packages/server/src/server/voice-permission-policy.ts
+4 -8
View File
@@ -1,7 +1,9 @@
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"];
const ALLOWED_TOKEN_SET = ["speak"];
const DENIED_TOKEN_SET = [
"mcp",
"paseo",
"bash",
"shell",
"terminal",
@@ -27,10 +29,7 @@ function stringifyMetadata(metadata: unknown): string {
}
}
/**
* Voice assistant policy: allow only MCP/paseo/speak tool requests.
* All non-tool permission requests are denied.
*/
/** Voice assistant policy: only allow the speak tool. */
export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean {
if (request.kind !== "tool") {
return false;
@@ -44,9 +43,6 @@ export function isVoicePermissionAllowed(request: AgentPermissionRequest): boole
if (normalizedName === "speak") {
return true;
}
if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) {
return true;
}
if (normalizedName !== "codextool") {
return false;
packages/server/src/server/voice-roundtrip.e2e.test.ts
+28 -10
View File
@@ -1,12 +1,15 @@
import { describe, test, expect, beforeAll, afterAll } from "vitest";
import { randomUUID } from "node:crypto";
import { Readable } from "node:stream";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import pino from "pino";
import {
createDaemonTestContext,
type DaemonTestContext,
} from "./test-utils/index.js";
import { getFullAccessConfig, type AgentProvider } from "./daemon-e2e/agent-configs.js";
import { OpenAITTS } from "./speech/providers/openai/tts.js";
import { OpenAISTT } from "./speech/providers/openai/stt.js";
import { STTManager } from "./agent/stt-manager.js";
@@ -88,8 +91,6 @@ describe("voice roundtrip e2e", () => {
voiceTts: { provider: "openai", explicit: true },
},
},
voiceLlmProvider: "claude",
voiceLlmProviderExplicit: true,
});
}, 60000);
@@ -97,9 +98,10 @@ describe("voice roundtrip e2e", () => {
await ctx.cleanup();
}, 60000);
speechTest(
"full roundtrip: voice input audio -> voice agent -> output audio -> transcribed output",
async () => {
for (const targetProvider of ["claude", "codex"] as const satisfies AgentProvider[]) {
speechTest(
`full roundtrip (${targetProvider}): voice input audio -> voice agent -> output audio -> transcribed output`,
async () => {
const logger = pino({ level: "silent" });
const ttsProvider = new OpenAITTS(
{
@@ -118,7 +120,20 @@ describe("voice roundtrip e2e", () => {
);
const sttOutput = new STTManager("voice-roundtrip-e2e", logger, sttProvider);
const voiceAgentId = randomUUID();
const voiceCwd = mkdtempSync(
path.join(tmpdir(), `voice-roundtrip-agent-${targetProvider}-`)
);
const voiceAgent = await withTimeout(
"createVoiceTargetAgent",
30000,
ctx.client.createAgent({
config: {
...getFullAccessConfig(targetProvider),
cwd: voiceCwd,
},
})
);
const voiceAgentId = voiceAgent.id;
const voiceMode = await withTimeout(
"setVoiceMode",
15000,
@@ -260,6 +275,8 @@ describe("voice roundtrip e2e", () => {
} finally {
offStream();
offErrors();
await ctx.client.setVoiceMode(false).catch(() => undefined);
rmSync(voiceCwd, { recursive: true, force: true });
}
})();
@@ -281,7 +298,8 @@ describe("voice roundtrip e2e", () => {
expect(normalized.length).toBeGreaterThan(0);
expect(normalized).toMatch(/round|trip|successful/);
},
180000
);
},
180000
);
}
});
packages/server/src/server/websocket-server.ts
-8
View File
@@ -83,10 +83,6 @@ export class VoiceAssistantWebSocketServer {
};
} | null;
private readonly voice: {
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
@@ -110,10 +106,6 @@ export class VoiceAssistantWebSocketServer {
speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
terminalManager?: TerminalManager | null,
voice?: {
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
packages/server/src/shared/messages.ts
+2 -2
View File
@@ -348,7 +348,7 @@ export const ArchiveAgentRequestMessageSchema = z.object({
export const SetVoiceModeMessageSchema = z.object({
type: z.literal("set_voice_mode"),
enabled: z.boolean(),
voiceAgentId: z.string().optional(),
agentId: z.string().optional(),
requestId: z.string().optional(),
});
@@ -578,7 +578,7 @@ export const SetVoiceModeResponseMessageSchema = z.object({
payload: z.object({
requestId: z.string(),
enabled: z.boolean(),
voiceAgentId: z.string().nullable(),
agentId: z.string().nullable(),
accepted: z.boolean(),
error: z.string().nullable(),
}),