jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: rename realtime to voice throughout codebase

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-03 23:12:17 +07:00
parent 4c5651fddc
commit d500b25632
16 changed files with 526 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/app/_layout.tsx
+3 -3
View File
@@ -5,7 +5,7 @@ import { KeyboardProvider } from "react-native-keyboard-controller";
import { GestureHandlerRootView, Gesture, GestureDetector } from "react-native-gesture-handler"; import { GestureHandlerRootView, Gesture, GestureDetector } from "react-native-gesture-handler";
import { BottomSheetModalProvider } from "@gorhom/bottom-sheet"; import { BottomSheetModalProvider } from "@gorhom/bottom-sheet";
import { PortalProvider } from "@gorhom/portal"; import { PortalProvider } from "@gorhom/portal";
import { RealtimeProvider } from "@/contexts/realtime-context"; import { VoiceProvider } from "@/contexts/voice-context";
import { useAppSettings } from "@/hooks/use-settings"; import { useAppSettings } from "@/hooks/use-settings";
import { useFaviconStatus } from "@/hooks/use-favicon-status"; import { useFaviconStatus } from "@/hooks/use-favicon-status";
import { View, ActivityIndicator, Text } from "react-native"; import { View, ActivityIndicator, Text } from "react-native";
@@ -226,10 +226,10 @@ function ProvidersWrapper({ children }: { children: ReactNode }) {
} }
return ( return (
<RealtimeProvider> <VoiceProvider>
<OfferLinkListener upsertDaemonFromOfferUrl={upsertDaemonFromOfferUrl} /> <OfferLinkListener upsertDaemonFromOfferUrl={upsertDaemonFromOfferUrl} />
{children} {children}
</RealtimeProvider> </VoiceProvider>
); );
} }
packages/app/src/components/agent-input-area.tsx
+62 -83
View File
@@ -1,26 +1,20 @@
import { import {
View, View,
Pressable, Pressable,
Platform,
Text, Text,
ActivityIndicator, ActivityIndicator,
} from "react-native"; } from "react-native";
import { useState, useEffect, useRef, useCallback, useMemo } from "react"; import { useState, useEffect, useRef, useCallback, useMemo } from "react";
import { StyleSheet, useUnistyles } from "react-native-unistyles"; import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { ArrowUp, AudioLines, Square, Pencil } from "lucide-react-native"; import { ArrowUp, AudioLines, MicOff, Square, Pencil } from "lucide-react-native";
import Animated, { import Animated, { useAnimatedStyle } from "react-native-reanimated";
useAnimatedStyle,
FadeIn,
FadeOut,
} from "react-native-reanimated";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
import { useSafeAreaInsets } from "react-native-safe-area-context"; import { useSafeAreaInsets } from "react-native-safe-area-context";
import { useRealtime } from "@/contexts/realtime-context"; import { useVoice } from "@/contexts/voice-context";
import { useIsFocused } from "@react-navigation/native"; import { useIsFocused } from "@react-navigation/native";
import { FOOTER_HEIGHT, MAX_CONTENT_WIDTH } from "@/constants/layout"; import { FOOTER_HEIGHT, MAX_CONTENT_WIDTH } from "@/constants/layout";
import { generateMessageId } from "@/types/stream"; import { generateMessageId } from "@/types/stream";
import { AgentStatusBar } from "./agent-status-bar"; import { AgentStatusBar } from "./agent-status-bar";
import { RealtimeControls } from "./realtime-controls";
import { useImageAttachmentPicker } from "@/hooks/use-image-attachment-picker"; import { useImageAttachmentPicker } from "@/hooks/use-image-attachment-picker";
import { useSessionStore } from "@/stores/session-store"; import { useSessionStore } from "@/stores/session-store";
import { useDraftStore } from "@/stores/draft-store"; import { useDraftStore } from "@/stores/draft-store";
@@ -57,15 +51,6 @@ interface AgentInputAreaProps {
} }
const EMPTY_ARRAY: readonly QueuedMessage[] = []; const EMPTY_ARRAY: readonly QueuedMessage[] = [];
// Android currently crashes inside ViewGroup.dispatchDraw when running Reanimated
// entering/exiting animations (see react-native-reanimated#8422), so guard them.
const SHOULD_DISABLE_ENTRY_EXIT_ANIMATIONS = Platform.OS === "android";
const REALTIME_FADE_IN = SHOULD_DISABLE_ENTRY_EXIT_ANIMATIONS
? undefined
: FadeIn.duration(250);
const REALTIME_FADE_OUT = SHOULD_DISABLE_ENTRY_EXIT_ANIMATIONS
? undefined
: FadeOut.duration(250);
export function AgentInputArea({ export function AgentInputArea({
agentId, agentId,
@@ -106,7 +91,7 @@ export function AgentInputArea({
const setQueuedMessages = useSessionStore((state) => state.setQueuedMessages); const setQueuedMessages = useSessionStore((state) => state.setQueuedMessages);
const { startRealtime, stopRealtime, isRealtimeMode } = useRealtime(); const { isVoiceMode, isMuted: isVoiceMuted, toggleMute: toggleVoiceMute } = useVoice();
const [internalInput, setInternalInput] = useState(""); const [internalInput, setInternalInput] = useState("");
const userInput = value ?? internalInput; const userInput = value ?? internalInput;
@@ -345,21 +330,6 @@ export function AgentInputArea({
}; };
}); });
async function handleRealtimePress() {
try {
if (isRealtimeMode) {
await stopRealtime();
} else {
if (!isConnected || !serverId) {
return;
}
await startRealtime(serverId);
}
} catch (error) {
console.error("[AgentInput] Failed to toggle realtime mode:", error);
}
}
function handleCancelAgent() { function handleCancelAgent() {
if (!agent || agent.status !== "running" || isCancellingAgent) { if (!agent || agent.status !== "running" || isCancellingAgent) {
return; return;
@@ -474,29 +444,7 @@ export function AgentInputArea({
<Square size={18} color="white" fill="white" /> <Square size={18} color="white" fill="white" />
)} )}
</Pressable> </Pressable>
) : ( ) : null;
<Pressable
onPress={handleRealtimePress}
disabled={!isConnected && !isRealtimeMode}
style={[
styles.realtimeButton as any,
(isRealtimeMode ? styles.realtimeButtonActive : undefined) as any,
(!isConnected && !isRealtimeMode
? styles.buttonDisabled
: undefined) as any,
]}
>
{isRealtimeMode ? (
<Square
size={18}
color={theme.colors.background}
fill={theme.colors.background}
/>
) : (
<AudioLines size={20} color={theme.colors.background} />
)}
</Pressable>
);
const leftContent = <AgentStatusBar agentId={agentId} serverId={serverId} />; const leftContent = <AgentStatusBar agentId={agentId} serverId={serverId} />;
@@ -508,17 +456,6 @@ export function AgentInputArea({
keyboardAnimatedStyle, keyboardAnimatedStyle,
]} ]}
> >
{/* Realtime controls - only when active */}
{isRealtimeMode && (
<Animated.View
style={styles.realtimeControlsContainer}
entering={REALTIME_FADE_IN}
exiting={REALTIME_FADE_OUT}
>
<RealtimeControls />
</Animated.View>
)}
{/* Input area */} {/* Input area */}
<View style={styles.inputAreaContainer}> <View style={styles.inputAreaContainer}>
<View style={styles.inputAreaContent}> <View style={styles.inputAreaContent}>
@@ -564,6 +501,35 @@ export function AgentInputArea({
/> />
)} )}
{/* Voice quick mute indicator */}
{isVoiceMode && (
<View style={styles.voiceIndicatorRow}>
<Pressable
onPress={toggleVoiceMute}
accessibilityRole="button"
accessibilityLabel={isVoiceMuted ? "Unmute voice" : "Mute voice"}
style={[
styles.voiceIndicatorPill,
isVoiceMuted && styles.voiceIndicatorPillMuted,
]}
>
{isVoiceMuted ? (
<MicOff size={14} color={theme.colors.surface0} />
) : (
<AudioLines size={14} color={theme.colors.foreground} />
)}
<Text
style={[
styles.voiceIndicatorText,
isVoiceMuted && styles.voiceIndicatorTextMuted,
]}
>
{isVoiceMuted ? "Voice muted" : "Voice on"}
</Text>
</Pressable>
</View>
)}
{/* MessageInput handles everything: text, dictation, attachments, all buttons */} {/* MessageInput handles everything: text, dictation, attachments, all buttons */}
<MessageInput <MessageInput
ref={messageInputRef} ref={messageInputRef}
@@ -578,7 +544,7 @@ export function AgentInputArea({
client={client} client={client}
placeholder="Message agent..." placeholder="Message agent..."
autoFocus={autoFocus} autoFocus={autoFocus}
disabled={isRealtimeMode || isSubmitLoading} disabled={isSubmitLoading}
isScreenFocused={isScreenFocused} isScreenFocused={isScreenFocused}
leftContent={leftContent} leftContent={leftContent}
rightContent={rightContent} rightContent={rightContent}
@@ -603,9 +569,6 @@ const styles = StyleSheet.create(((theme: Theme) => ({
height: theme.borderWidth[1], height: theme.borderWidth[1],
backgroundColor: theme.colors.border, backgroundColor: theme.colors.border,
}, },
realtimeControlsContainer: {
height: FOOTER_HEIGHT,
},
inputAreaContainer: { inputAreaContainer: {
position: "relative", position: "relative",
minHeight: FOOTER_HEIGHT, minHeight: FOOTER_HEIGHT,
@@ -620,17 +583,6 @@ const styles = StyleSheet.create(((theme: Theme) => ({
maxWidth: MAX_CONTENT_WIDTH, maxWidth: MAX_CONTENT_WIDTH,
gap: theme.spacing[3], gap: theme.spacing[3],
}, },
realtimeButton: {
width: 34,
height: 34,
borderRadius: theme.borderRadius.full,
backgroundColor: theme.colors.accentForeground,
alignItems: "center",
justifyContent: "center",
},
realtimeButtonActive: {
backgroundColor: theme.colors.palette.blue[600],
},
cancelButton: { cancelButton: {
width: 34, width: 34,
height: 34, height: 34,
@@ -642,6 +594,33 @@ const styles = StyleSheet.create(((theme: Theme) => ({
buttonDisabled: { buttonDisabled: {
opacity: 0.5, opacity: 0.5,
}, },
voiceIndicatorRow: {
flexDirection: "row",
justifyContent: "flex-end",
},
voiceIndicatorPill: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[2],
paddingHorizontal: theme.spacing[3],
height: 32,
borderRadius: theme.borderRadius.full,
backgroundColor: theme.colors.surface2,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
},
voiceIndicatorPillMuted: {
backgroundColor: theme.colors.palette.red[600],
borderColor: theme.colors.palette.red[800],
},
voiceIndicatorText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.xs,
fontWeight: theme.fontWeight.medium,
},
voiceIndicatorTextMuted: {
color: theme.colors.surface0,
},
queueContainer: { queueContainer: {
flexDirection: "column", flexDirection: "column",
gap: theme.spacing[2], gap: theme.spacing[2],
packages/app/src/components/home-footer.tsx
-297
View File
@@ -1,297 +0,0 @@
import { useCallback, useMemo, useState } from "react";
import { View, Pressable, Text, Platform, Modal, Alert } from "react-native";
import { useRouter } from "expo-router";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { AudioLines, Users, Plus } from "lucide-react-native";
import { useRealtime } from "@/contexts/realtime-context";
import { useDaemonConnections } from "@/contexts/daemon-connections-context";
import { FOOTER_HEIGHT } from "@/constants/layout";
import { RealtimeControls } from "./realtime-controls";
import Animated, {
FadeIn,
FadeOut,
useAnimatedStyle,
} from "react-native-reanimated";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
export function HomeFooter() {
const { theme } = useUnistyles();
const insets = useSafeAreaInsets();
const router = useRouter();
const { isRealtimeMode, startRealtime } = useRealtime();
const { connectionStates } = useDaemonConnections();
const [showRealtimeHostPicker, setShowRealtimeHostPicker] = useState(false);
// Guard Reanimated entry/exit transitions on Android to avoid ViewGroup.dispatchDraw crashes
// tracked in react-native-reanimated#8422.
const shouldDisableEntryExitAnimations = Platform.OS === "android";
const realtimeFadeIn = shouldDisableEntryExitAnimations ? undefined : FadeIn.duration(250);
const realtimeFadeOut = shouldDisableEntryExitAnimations ? undefined : FadeOut.duration(250);
const { height: keyboardHeight } = useReanimatedKeyboardAnimation();
const bottomInset = insets.bottom;
const keyboardAnimatedStyle = useAnimatedStyle(
() => {
"worklet";
const absoluteHeight = Math.abs(keyboardHeight.value);
const shift = Math.max(0, absoluteHeight - bottomInset);
return {
transform: [{ translateY: -shift }],
};
},
[bottomInset],
);
const realtimeEligibleHosts = useMemo(() => {
return Array.from(connectionStates.values()).filter((entry) => entry.status === "online");
}, [connectionStates]);
const hasAnyConfiguredHosts = connectionStates.size > 0;
const handleStartRealtime = useCallback(() => {
if (realtimeEligibleHosts.length === 0) {
if (!hasAnyConfiguredHosts) {
Alert.alert(
"No hosts available",
"Add a host in Settings before starting realtime mode."
);
return;
}
Alert.alert(
"Hosts reconnecting",
"Every host is offline right now. Paseo reconnects automatically—try realtime again once one comes online."
);
return;
}
if (realtimeEligibleHosts.length === 1) {
void startRealtime(realtimeEligibleHosts[0].daemon.id).catch((error) => {
console.error("[HomeFooter] Failed to start realtime", error);
Alert.alert("Realtime failed", "Unable to start realtime mode for this host.");
});
return;
}
setShowRealtimeHostPicker(true);
}, [hasAnyConfiguredHosts, realtimeEligibleHosts, startRealtime]);
const handleSelectRealtimeHost = useCallback(
(daemonId: string) => {
setShowRealtimeHostPicker(false);
void startRealtime(daemonId).catch((error) => {
console.error("[HomeFooter] Failed to start realtime", error);
Alert.alert("Realtime failed", "Unable to start realtime mode for this host.");
});
},
[startRealtime]
);
const handleDismissHostPicker = useCallback(() => {
setShowRealtimeHostPicker(false);
}, []);
// For home and orchestrator screens, show action buttons with realtime stacked on top
const nonAgentFooterHeight = isRealtimeMode
? FOOTER_HEIGHT * 2 + insets.bottom
: FOOTER_HEIGHT + insets.bottom;
const iconSize = 24;
const iconStyle = { width: iconSize, height: iconSize };
return (
<>
<Animated.View
style={[
styles.container,
{
paddingBottom: insets.bottom,
height: nonAgentFooterHeight,
},
keyboardAnimatedStyle,
]}
>
<View style={styles.nonAgentContent}>
{/* Realtime controls - only visible when active */}
{isRealtimeMode && (
<Animated.View
style={styles.realtimeSection}
entering={realtimeFadeIn}
exiting={realtimeFadeOut}
>
<RealtimeControls />
</Animated.View>
)}
{/* Action menu */}
<View style={styles.actionButtonContainer}>
<Pressable
onPress={() => router.push("/agents")}
style={({ pressed }) => [
styles.footerButton,
pressed && styles.buttonPressed,
]}
>
<View style={styles.footerIconWrapper}>
<Users
size={iconSize}
color={theme.colors.foreground}
style={iconStyle}
/>
</View>
<Text style={styles.footerButtonText}>Agents</Text>
</Pressable>
<Pressable
onPress={() => {
console.log("[HomeFooter] New Agent button pressed");
router.push("/agent" as any);
}}
style={({ pressed }) => [
styles.footerButton,
pressed && styles.buttonPressed,
]}
>
<View style={styles.footerIconWrapper}>
<Plus
size={iconSize}
color={theme.colors.foreground}
style={iconStyle}
/>
</View>
<Text style={styles.footerButtonText}>New agent</Text>
</Pressable>
<Pressable
onPress={handleStartRealtime}
disabled={realtimeEligibleHosts.length === 0}
style={({ pressed }) => [
styles.footerButton,
realtimeEligibleHosts.length === 0 && styles.buttonDisabled,
pressed && realtimeEligibleHosts.length === 0 && styles.buttonPressed,
]}
>
<View style={styles.footerIconWrapper}>
<AudioLines
size={iconSize}
color={theme.colors.foreground}
style={iconStyle}
/>
</View>
<Text style={styles.footerButtonText}>Realtime</Text>
</Pressable>
</View>
</View>
</Animated.View>
<Modal
visible={showRealtimeHostPicker}
transparent
animationType="fade"
onRequestClose={handleDismissHostPicker}
>
<View style={styles.hostPickerOverlay}>
<Pressable style={styles.hostPickerBackdrop} onPress={handleDismissHostPicker} />
<View style={styles.hostPickerContainer}>
<Text style={styles.hostPickerTitle}>Choose a host</Text>
{realtimeEligibleHosts.map((entry) => (
<Pressable
key={entry.daemon.id}
style={styles.hostPickerButton}
onPress={() => handleSelectRealtimeHost(entry.daemon.id)}
>
<Text style={styles.hostPickerButtonText}>{entry.daemon.label}</Text>
</Pressable>
))}
<Pressable style={styles.hostPickerCancel} onPress={handleDismissHostPicker}>
<Text style={styles.hostPickerCancelText}>Cancel</Text>
</Pressable>
</View>
</View>
</Modal>
</>
);
}
const styles = StyleSheet.create((theme) => ({
container: {
backgroundColor: theme.colors.surface0,
borderTopWidth: theme.borderWidth[1],
borderTopColor: theme.colors.border,
},
nonAgentContent: {
flexDirection: "column",
},
realtimeSection: {
height: FOOTER_HEIGHT,
borderBottomWidth: theme.borderWidth[1],
borderBottomColor: theme.colors.border,
},
actionButtonContainer: {
flexDirection: "row",
padding: theme.spacing[4],
gap: theme.spacing[3],
height: FOOTER_HEIGHT,
},
footerButton: {
flex: 1,
flexDirection: "column",
alignItems: "center",
justifyContent: "center",
paddingVertical: theme.spacing[3],
gap: theme.spacing[1],
},
footerIconWrapper: {
width: 28,
height: 28,
alignItems: "center",
justifyContent: "center",
},
footerButtonText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.xs,
fontWeight: theme.fontWeight.normal,
},
buttonDisabled: {
opacity: 0.5,
},
buttonPressed: {
opacity: 0.5,
},
hostPickerOverlay: {
flex: 1,
backgroundColor: "rgba(0,0,0,0.5)",
justifyContent: "flex-end",
},
hostPickerBackdrop: {
flex: 1,
},
hostPickerContainer: {
backgroundColor: theme.colors.surface2,
padding: theme.spacing[4],
borderTopLeftRadius: theme.borderRadius.xl,
borderTopRightRadius: theme.borderRadius.xl,
gap: theme.spacing[3],
},
hostPickerTitle: {
fontSize: theme.fontSize.lg,
fontWeight: theme.fontWeight.semibold,
color: theme.colors.foreground,
textAlign: "center",
},
hostPickerButton: {
paddingVertical: theme.spacing[3],
borderRadius: theme.borderRadius.lg,
backgroundColor: theme.colors.surface2,
},
hostPickerButtonText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.sm,
textAlign: "center",
},
hostPickerCancel: {
paddingVertical: theme.spacing[3],
},
hostPickerCancelText: {
color: theme.colors.foregroundMuted,
fontSize: theme.fontSize.sm,
textAlign: "center",
},
}));
packages/app/src/components/message-input.tsx
+6 -1
View File
@@ -28,6 +28,7 @@ import { useDictation } from "@/hooks/use-dictation";
import { DictationOverlay } from "./dictation-controls"; import { DictationOverlay } from "./dictation-controls";
import type { DaemonClientV2 } from "@server/client/daemon-client-v2"; import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
import { usePanelStore } from "@/stores/panel-store"; import { usePanelStore } from "@/stores/panel-store";
import { useVoiceOptional } from "@/contexts/voice-context";
export interface ImageAttachment { export interface ImageAttachment {
uri: string; uri: string;
@@ -118,6 +119,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
ref ref
) { ) {
const { theme } = useUnistyles(); const { theme } = useUnistyles();
const voice = useVoiceOptional();
const toggleAgentList = usePanelStore((state) => state.toggleAgentList); const toggleAgentList = usePanelStore((state) => state.toggleAgentList);
const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT); const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT);
const textInputRef = useRef< const textInputRef = useRef<
@@ -294,9 +296,12 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
if (isDictating) { if (isDictating) {
await cancelDictation(); await cancelDictation();
} else { } else {
if (voice?.isVoiceMode) {
await voice.stopVoice();
}
await startDictation(); await startDictation();
} }
}, [isDictating, cancelDictation, startDictation]); }, [isDictating, cancelDictation, startDictation, voice]);
const handleCancelRecording = useCallback(async () => { const handleCancelRecording = useCallback(async () => {
await cancelDictation(); await cancelDictation();
packages/app/src/components/realtime-controls.tsx
-110
View File
@@ -1,110 +0,0 @@
import { View, Text, Pressable } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { MicOff, Square } from "lucide-react-native";
import { VolumeMeter } from "./volume-meter";
import { useRealtime } from "@/contexts/realtime-context";
import { FOOTER_HEIGHT } from "@/constants/layout";
const CONTROL_BUTTON_SIZE = 48;
const VERTICAL_PADDING = (FOOTER_HEIGHT - CONTROL_BUTTON_SIZE) / 2;
export function RealtimeControls() {
const { theme } = useUnistyles();
const {
volume,
isMuted,
isDetecting,
isSpeaking,
segmentDuration,
stopRealtime,
toggleMute,
} = useRealtime();
function handleStop() {
stopRealtime();
}
return (
<View style={styles.container}>
<View style={styles.volumeContainer}>
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
/>
</View>
<View style={styles.buttons}>
{/* Mute button */}
<Pressable
onPress={toggleMute}
style={[
styles.muteButton,
isMuted && styles.muteButtonActive,
]}
>
<MicOff
size={20}
color={
isMuted
? theme.colors.surface0
: theme.colors.foreground
}
/>
</Pressable>
{/* Stop button */}
<Pressable
onPress={handleStop}
style={styles.stopButton}
>
<Square size={18} color="white" fill="white" />
</Pressable>
</View>
</View>
);
}
const styles = StyleSheet.create((theme) => ({
container: {
flexDirection: "row",
alignItems: "center",
justifyContent: "space-between",
paddingHorizontal: theme.spacing[4],
paddingVertical: VERTICAL_PADDING,
height: FOOTER_HEIGHT,
},
volumeContainer: {
flex: 1,
justifyContent: "center",
alignItems: "flex-start",
},
buttons: {
flexDirection: "row",
alignItems: "center",
justifyContent: "center",
gap: theme.spacing[3],
},
muteButton: {
width: 48,
height: 48,
borderRadius: theme.borderRadius.full,
alignItems: "center",
justifyContent: "center",
backgroundColor: theme.colors.surface2,
borderWidth: theme.borderWidth[2],
borderColor: theme.colors.border,
},
muteButtonActive: {
backgroundColor: theme.colors.palette.red[500],
borderColor: theme.colors.palette.red[600],
},
stopButton: {
width: 48,
height: 48,
borderRadius: theme.borderRadius.full,
alignItems: "center",
justifyContent: "center",
backgroundColor: theme.colors.palette.red[600],
},
}));
packages/app/src/components/sliding-sidebar.tsx
+199 -10
View File
@@ -1,5 +1,5 @@
import { useCallback, useMemo, useState, useEffect } from "react"; import { useCallback, useMemo, useState, useEffect } from "react";
import { View, Pressable, Text, Platform } from "react-native"; import { View, Pressable, Text, Platform, Modal, Alert } from "react-native";
import { useSafeAreaInsets } from "react-native-safe-area-context"; import { useSafeAreaInsets } from "react-native-safe-area-context";
import Animated, { import Animated, {
useAnimatedStyle, useAnimatedStyle,
@@ -9,13 +9,16 @@ import Animated, {
} from "react-native-reanimated"; } from "react-native-reanimated";
import { Gesture, GestureDetector } from "react-native-gesture-handler"; import { Gesture, GestureDetector } from "react-native-gesture-handler";
import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles"; import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles";
import { Plus, Settings, Users } from "lucide-react-native"; import { Plus, Settings, Users, AudioLines } from "lucide-react-native";
import { router } from "expo-router"; import { router } from "expo-router";
import { usePanelStore } from "@/stores/panel-store"; import { usePanelStore } from "@/stores/panel-store";
import { GroupedAgentList } from "./grouped-agent-list"; import { GroupedAgentList } from "./grouped-agent-list";
import { useAggregatedAgents } from "@/hooks/use-aggregated-agents"; import { useAggregatedAgents } from "@/hooks/use-aggregated-agents";
import { useSidebarAnimation } from "@/contexts/sidebar-animation-context"; import { useSidebarAnimation } from "@/contexts/sidebar-animation-context";
import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window"; import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window";
import { useVoice } from "@/contexts/voice-context";
import { useDaemonConnections } from "@/contexts/daemon-connections-context";
import { VoicePanel } from "./voice-panel";
const DESKTOP_SIDEBAR_WIDTH = 320; const DESKTOP_SIDEBAR_WIDTH = 320;
const SIDEBAR_AGENT_LIMIT = 15; const SIDEBAR_AGENT_LIMIT = 15;
@@ -48,6 +51,9 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
} = useSidebarAnimation(); } = useSidebarAnimation();
const trafficLightPadding = useTrafficLightPadding(); const trafficLightPadding = useTrafficLightPadding();
const dragHandlers = useTauriDragHandlers(); const dragHandlers = useTauriDragHandlers();
const { connectionStates } = useDaemonConnections();
const { isVoiceMode, startVoice, stopVoice } = useVoice();
const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false);
// Track user-initiated refresh to avoid showing spinner on background revalidation // Track user-initiated refresh to avoid showing spinner on background revalidation
const [isManualRefresh, setIsManualRefresh] = useState(false); const [isManualRefresh, setIsManualRefresh] = useState(false);
@@ -124,6 +130,62 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
router.push("/agents"); router.push("/agents");
}, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]); }, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]);
const voiceEligibleHosts = useMemo(() => {
return Array.from(connectionStates.values()).filter((entry) => entry.status === "online");
}, [connectionStates]);
const hasAnyConfiguredHosts = connectionStates.size > 0;
const handleToggleVoice = useCallback(() => {
if (isVoiceMode) {
void stopVoice().catch((error) => {
console.error("[SlidingSidebar] Failed to stop voice", error);
Alert.alert("Voice failed", "Unable to stop voice mode.");
});
return;
}
if (voiceEligibleHosts.length === 0) {
if (!hasAnyConfiguredHosts) {
Alert.alert(
"No hosts available",
"Add a host in Settings before starting Voice mode.",
[{ text: "Open Settings", onPress: () => router.push("/settings") }, { text: "OK" }]
);
return;
}
Alert.alert(
"Hosts reconnecting",
"Every host is offline right now. Paseo reconnects automatically—try Voice mode again once one comes online."
);
return;
}
if (voiceEligibleHosts.length === 1) {
void startVoice(voiceEligibleHosts[0].daemon.id).catch((error) => {
console.error("[SlidingSidebar] Failed to start voice", error);
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
});
return;
}
setShowVoiceHostPicker(true);
}, [hasAnyConfiguredHosts, isVoiceMode, startVoice, stopVoice, voiceEligibleHosts]);
const handleSelectVoiceHost = useCallback(
(daemonId: string) => {
setShowVoiceHostPicker(false);
void startVoice(daemonId).catch((error) => {
console.error("[SlidingSidebar] Failed to start voice", error);
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
});
},
[startVoice]
);
const handleDismissVoiceHostPicker = useCallback(() => {
setShowVoiceHostPicker(false);
}, []);
// Close gesture (swipe left to close when sidebar is open) // Close gesture (swipe left to close when sidebar is open)
// Only activates on leftward swipe, fails on rightward or vertical movement // Only activates on leftward swipe, fails on rightward or vertical movement
// This mirrors the explorer-sidebar pattern for the right sidebar // This mirrors the explorer-sidebar pattern for the right sidebar
@@ -224,6 +286,8 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
parentGestureRef={closeGestureRef} parentGestureRef={closeGestureRef}
/> />
{isVoiceMode ? <VoicePanel /> : null}
{/* Footer */} {/* Footer */}
<View style={styles.sidebarFooter}> <View style={styles.sidebarFooter}>
<Pressable <Pressable
@@ -239,6 +303,25 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
</> </>
)} )}
</Pressable> </Pressable>
<View style={styles.footerIconRow}>
<Pressable
style={styles.footerIconButton}
testID="sidebar-voice"
onPress={handleToggleVoice}
>
{({ hovered }) => (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)}
</Pressable>
<Pressable <Pressable
style={styles.footerIconButton} style={styles.footerIconButton}
onPress={handleSettingsMobile} onPress={handleSettingsMobile}
@@ -247,10 +330,37 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
<Settings size={20} color={hovered ? theme.colors.foreground : theme.colors.foregroundMuted} /> <Settings size={20} color={hovered ? theme.colors.foreground : theme.colors.foregroundMuted} />
)} )}
</Pressable> </Pressable>
</View>
</View> </View>
</View> </View>
</Animated.View> </Animated.View>
</GestureDetector> </GestureDetector>
<Modal
visible={showVoiceHostPicker}
transparent
animationType="fade"
onRequestClose={handleDismissVoiceHostPicker}
>
<View style={styles.hostPickerOverlay}>
<Pressable style={styles.hostPickerBackdrop} onPress={handleDismissVoiceHostPicker} />
<View style={styles.hostPickerContainer}>
<Text style={styles.hostPickerTitle}>Choose a host</Text>
{voiceEligibleHosts.map((entry) => (
<Pressable
key={entry.daemon.id}
style={styles.hostPickerButton}
onPress={() => handleSelectVoiceHost(entry.daemon.id)}
>
<Text style={styles.hostPickerButtonText}>{entry.daemon.label}</Text>
</Pressable>
))}
<Pressable style={styles.hostPickerCancel} onPress={handleDismissVoiceHostPicker}>
<Text style={styles.hostPickerCancelText}>Cancel</Text>
</Pressable>
</View>
</View>
</Modal>
</View> </View>
); );
} }
@@ -289,6 +399,8 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
selectedAgentId={selectedAgentId} selectedAgentId={selectedAgentId}
/> />
{isVoiceMode ? <VoicePanel /> : null}
{/* Footer */} {/* Footer */}
<View style={styles.sidebarFooter}> <View style={styles.sidebarFooter}>
<Pressable <Pressable
@@ -304,14 +416,34 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
</> </>
)} )}
</Pressable> </Pressable>
<Pressable <View style={styles.footerIconRow}>
style={styles.footerIconButton} <Pressable
onPress={handleSettingsDesktop} style={styles.footerIconButton}
> testID="sidebar-voice"
{({ hovered }) => ( onPress={handleToggleVoice}
<Settings size={20} color={hovered ? theme.colors.foreground : theme.colors.foregroundMuted} /> >
)} {({ hovered }) => (
</Pressable> <AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)}
</Pressable>
<Pressable
style={styles.footerIconButton}
onPress={handleSettingsDesktop}
>
{({ hovered }) => (
<Settings size={20} color={hovered ? theme.colors.foreground : theme.colors.foregroundMuted} />
)}
</Pressable>
</View>
</View> </View>
</View> </View>
); );
@@ -376,6 +508,11 @@ const styles = StyleSheet.create((theme) => ({
borderTopWidth: 1, borderTopWidth: 1,
borderTopColor: theme.colors.border, borderTopColor: theme.colors.border,
}, },
footerIconRow: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[3],
},
footerButton: { footerButton: {
flexDirection: "row", flexDirection: "row",
alignItems: "center", alignItems: "center",
@@ -395,4 +532,56 @@ const styles = StyleSheet.create((theme) => ({
footerButtonTextHovered: { footerButtonTextHovered: {
color: theme.colors.foreground, color: theme.colors.foreground,
}, },
hostPickerOverlay: {
flex: 1,
justifyContent: "center",
alignItems: "center",
backgroundColor: "rgba(0, 0, 0, 0.5)",
padding: theme.spacing[4],
},
hostPickerBackdrop: {
...StyleSheet.absoluteFillObject,
},
hostPickerContainer: {
width: "100%",
maxWidth: 360,
backgroundColor: theme.colors.surface0,
borderRadius: theme.borderRadius["2xl"],
padding: theme.spacing[4],
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
gap: theme.spacing[2],
},
hostPickerTitle: {
fontSize: theme.fontSize.base,
fontWeight: theme.fontWeight.semibold,
color: theme.colors.foreground,
marginBottom: theme.spacing[2],
},
hostPickerButton: {
paddingVertical: theme.spacing[3],
paddingHorizontal: theme.spacing[3],
borderRadius: theme.borderRadius.lg,
backgroundColor: theme.colors.surface2,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
},
hostPickerButtonText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.sm,
},
hostPickerCancel: {
marginTop: theme.spacing[2],
paddingVertical: theme.spacing[3],
paddingHorizontal: theme.spacing[3],
borderRadius: theme.borderRadius.lg,
backgroundColor: theme.colors.surface0,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
alignItems: "center",
},
hostPickerCancelText: {
color: theme.colors.foregroundMuted,
fontSize: theme.fontSize.sm,
},
})); }));
packages/app/src/components/voice-panel.tsx
+139
View File
@@ -0,0 +1,139 @@
import { View, Text, Pressable } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { MicOff, Square, AudioLines } from "lucide-react-native";
import { VolumeMeter } from "./volume-meter";
import { useVoice } from "@/contexts/voice-context";
import { useDaemonConnections } from "@/contexts/daemon-connections-context";
export function VoicePanel() {
const { theme } = useUnistyles();
const { connectionStates } = useDaemonConnections();
const {
volume,
isMuted,
isDetecting,
isSpeaking,
stopVoice,
toggleMute,
activeServerId,
} = useVoice();
const activeHost = activeServerId ? connectionStates.get(activeServerId) ?? null : null;
const hostLabel = activeHost?.daemon.label ?? null;
const hostStatus = activeHost?.status ?? null;
return (
<View style={styles.container}>
<View style={styles.headerRow}>
<View style={styles.titleRow}>
<AudioLines size={16} color={theme.colors.foregroundMuted} />
<Text style={styles.titleText}>Voice</Text>
</View>
<Text style={styles.hostText} numberOfLines={1}>
{hostLabel ? `Host: ${hostLabel}` : "Host: unknown"}
{hostStatus ? ` (${hostStatus})` : ""}
</Text>
</View>
<View style={styles.meterRow}>
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
/>
</View>
<View style={styles.actionsRow}>
<Pressable
onPress={toggleMute}
accessibilityRole="button"
accessibilityLabel={isMuted ? "Unmute voice" : "Mute voice"}
style={[
styles.iconButton,
isMuted && styles.iconButtonMuted,
]}
>
<MicOff
size={18}
color={isMuted ? theme.colors.surface0 : theme.colors.foreground}
/>
</Pressable>
<Pressable
onPress={() => void stopVoice()}
accessibilityRole="button"
accessibilityLabel="Stop voice mode"
style={[styles.iconButton, styles.iconButtonStop]}
>
<Square size={16} color="white" fill="white" />
</Pressable>
</View>
</View>
);
}
const styles = StyleSheet.create((theme) => ({
container: {
marginHorizontal: theme.spacing[4],
marginBottom: theme.spacing[3],
borderRadius: theme.borderRadius["2xl"],
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
backgroundColor: theme.colors.surface2,
paddingVertical: theme.spacing[3],
paddingHorizontal: theme.spacing[3],
gap: theme.spacing[3],
},
headerRow: {
flexDirection: "row",
alignItems: "center",
justifyContent: "space-between",
gap: theme.spacing[3],
},
titleRow: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[2],
flexShrink: 0,
},
titleText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.sm,
fontWeight: theme.fontWeight.semibold,
},
hostText: {
flex: 1,
textAlign: "right",
color: theme.colors.foregroundMuted,
fontSize: theme.fontSize.xs,
},
meterRow: {
justifyContent: "center",
},
actionsRow: {
flexDirection: "row",
alignItems: "center",
justifyContent: "flex-end",
gap: theme.spacing[2],
},
iconButton: {
width: 40,
height: 40,
borderRadius: theme.borderRadius.full,
alignItems: "center",
justifyContent: "center",
backgroundColor: theme.colors.surface0,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
},
iconButtonMuted: {
backgroundColor: theme.colors.palette.red[500],
borderColor: theme.colors.palette.red[600],
},
iconButtonStop: {
backgroundColor: theme.colors.palette.red[600],
borderColor: theme.colors.palette.red[800],
},
}));
packages/app/src/contexts/realtime-context.tsx → packages/app/src/contexts/voice-context.tsx
+42 -38
View File
@@ -7,34 +7,38 @@ import { randomUUID } from "expo-crypto";
const VOICE_CONVERSATION_ID_STORAGE_KEY = "@paseo:voice-conversation-id"; const VOICE_CONVERSATION_ID_STORAGE_KEY = "@paseo:voice-conversation-id";
interface RealtimeContextValue { interface VoiceContextValue {
isRealtimeMode: boolean; isVoiceMode: boolean;
volume: number; volume: number;
isMuted: boolean; isMuted: boolean;
isDetecting: boolean; isDetecting: boolean;
isSpeaking: boolean; isSpeaking: boolean;
segmentDuration: number; segmentDuration: number;
startRealtime: (serverId: string) => Promise<void>; startVoice: (serverId: string) => Promise<void>;
stopRealtime: () => Promise<void>; stopVoice: () => Promise<void>;
toggleMute: () => void; toggleMute: () => void;
activeServerId: string | null; activeServerId: string | null;
} }
const RealtimeContext = createContext<RealtimeContextValue | null>(null); const VoiceContext = createContext<VoiceContextValue | null>(null);
export function useRealtime() { export function useVoice() {
const context = useContext(RealtimeContext); const context = useContext(VoiceContext);
if (!context) { if (!context) {
throw new Error("useRealtime must be used within RealtimeProvider"); throw new Error("useVoice must be used within VoiceProvider");
} }
return context; return context;
} }
interface RealtimeProviderProps { export function useVoiceOptional(): VoiceContextValue | null {
return useContext(VoiceContext);
}
interface VoiceProviderProps {
children: ReactNode; children: ReactNode;
} }
export function RealtimeProvider({ children }: RealtimeProviderProps) { export function VoiceProvider({ children }: VoiceProviderProps) {
const getSession = useSessionStore((state) => state.getSession); const getSession = useSessionStore((state) => state.getSession);
const [activeServerId, setActiveServerId] = useState<string | null>(null); const [activeServerId, setActiveServerId] = useState<string | null>(null);
const activeSession = useSessionStore( const activeSession = useSessionStore(
@@ -49,12 +53,12 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
) )
); );
const realtimeSessionRef = useRef<SessionState | null>(null); const realtimeSessionRef = useRef<SessionState | null>(null);
const [isRealtimeMode, setIsRealtimeMode] = useState(false); const [isVoiceMode, setIsVoiceMode] = useState(false);
const bargeInPlaybackStopRef = useRef<number | null>(null); const bargeInPlaybackStopRef = useRef<number | null>(null);
const realtimeAudio = useSpeechmaticsAudio({ const realtimeAudio = useSpeechmaticsAudio({
onSpeechStart: () => { onSpeechStart: () => {
console.log("[Realtime] Speech detected"); console.log("[Voice] Speech detected");
// Stop audio playback if playing // Stop audio playback if playing
const session = realtimeSessionRef.current; const session = realtimeSessionRef.current;
const sessionAudioPlayer = session?.audioPlayer ?? null; const sessionAudioPlayer = session?.audioPlayer ?? null;
@@ -72,20 +76,20 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
try { try {
if (sessionClient) { if (sessionClient) {
void sessionClient.abortRequest().catch((error) => { void sessionClient.abortRequest().catch((error) => {
console.error("[Realtime] Failed to send abort_request:", error); console.error("[Voice] Failed to send abort_request:", error);
}); });
} }
console.log("[Realtime] Sent abort_request before streaming audio"); console.log("[Voice] Sent abort_request before streaming audio");
} catch (error) { } catch (error) {
console.error("[Realtime] Failed to send abort_request:", error); console.error("[Voice] Failed to send abort_request:", error);
} }
}, },
onSpeechEnd: () => { onSpeechEnd: () => {
console.log("[Realtime] Speech ended"); console.log("[Voice] Speech ended");
}, },
onAudioSegment: ({ audioData, isLast }) => { onAudioSegment: ({ audioData, isLast }) => {
console.log( console.log(
"[Realtime] Sending audio segment, length:", "[Voice] Sending audio segment, length:",
audioData.length, audioData.length,
"isLast:", "isLast:",
isLast isLast
@@ -96,25 +100,25 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
try { try {
if (session?.client) { if (session?.client) {
void session.client void session.client
.sendRealtimeAudioChunk( .sendVoiceAudioChunk(
audioData, audioData,
"audio/pcm;rate=16000;bits=16", "audio/pcm;rate=16000;bits=16",
isLast isLast
) )
.catch((error) => { .catch((error) => {
console.error("[Realtime] Failed to send audio segment:", error); console.error("[Voice] Failed to send audio segment:", error);
}); });
} }
} catch (error) { } catch (error) {
console.error("[Realtime] Failed to send audio segment:", error); console.error("[Voice] Failed to send audio segment:", error);
} }
}, },
onError: (error) => { onError: (error) => {
console.error("[Realtime] Audio error:", error); console.error("[Voice] Audio error:", error);
const session = realtimeSessionRef.current; const session = realtimeSessionRef.current;
if (session?.client) { if (session?.client) {
// Send error through websocket instead of directly manipulating messages // Send error through websocket instead of directly manipulating messages
console.error("[Realtime] Cannot handle error - setMessages not available from SessionState"); console.error("[Voice] Cannot handle error - setMessages not available from SessionState");
} }
}, },
volumeThreshold: 0.3, volumeThreshold: 0.3,
@@ -149,7 +153,7 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
} }
}, [isPlayingAudio]); }, [isPlayingAudio]);
const startRealtime = useCallback( const startVoice = useCallback(
async (serverId: string) => { async (serverId: string) => {
const session = getSession(serverId) ?? null; const session = getSession(serverId) ?? null;
if (!session) { if (!session) {
@@ -160,8 +164,8 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
realtimeSessionRef.current = session; realtimeSessionRef.current = session;
setActiveServerId(serverId); setActiveServerId(serverId);
await realtimeAudio.start(); await realtimeAudio.start();
setIsRealtimeMode(true); setIsVoiceMode(true);
console.log("[Realtime] Mode enabled"); console.log("[Voice] Mode enabled");
if (session?.client) { if (session?.client) {
let voiceConversationId = let voiceConversationId =
@@ -175,10 +179,10 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
} }
await session.client.setVoiceConversation(true, voiceConversationId); await session.client.setVoiceConversation(true, voiceConversationId);
} else { } else {
console.warn("[Realtime] setRealtimeMode skipped: daemon unavailable"); console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
} }
} catch (error: any) { } catch (error: any) {
console.error("[Realtime] Failed to start:", error); console.error("[Voice] Failed to start:", error);
setActiveServerId((current) => (current === serverId ? null : current)); setActiveServerId((current) => (current === serverId ? null : current));
throw error; throw error;
} }
@@ -186,42 +190,42 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
[getSession, realtimeAudio] [getSession, realtimeAudio]
); );
const stopRealtime = useCallback(async () => { const stopVoice = useCallback(async () => {
try { try {
const session = realtimeSessionRef.current; const session = realtimeSessionRef.current;
session?.audioPlayer?.stop(); session?.audioPlayer?.stop();
await realtimeAudio.stop(); await realtimeAudio.stop();
setIsRealtimeMode(false); setIsVoiceMode(false);
setActiveServerId(null); setActiveServerId(null);
console.log("[Realtime] Mode disabled"); console.log("[Voice] Mode disabled");
if (session?.client) { if (session?.client) {
await session.client.setVoiceConversation(false); await session.client.setVoiceConversation(false);
} else { } else {
console.warn("[Realtime] setRealtimeMode skipped: daemon unavailable"); console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
} }
} catch (error: any) { } catch (error: any) {
console.error("[Realtime] Failed to stop:", error); console.error("[Voice] Failed to stop:", error);
throw error; throw error;
} }
}, [realtimeAudio]); }, [realtimeAudio]);
const value: RealtimeContextValue = { const value: VoiceContextValue = {
isRealtimeMode, isVoiceMode,
volume: realtimeAudio.volume, volume: realtimeAudio.volume,
isMuted: realtimeAudio.isMuted, isMuted: realtimeAudio.isMuted,
isDetecting: realtimeAudio.isDetecting, isDetecting: realtimeAudio.isDetecting,
isSpeaking: realtimeAudio.isSpeaking, isSpeaking: realtimeAudio.isSpeaking,
segmentDuration: realtimeAudio.segmentDuration, segmentDuration: realtimeAudio.segmentDuration,
startRealtime, startVoice,
stopRealtime, stopVoice,
toggleMute: realtimeAudio.toggleMute, toggleMute: realtimeAudio.toggleMute,
activeServerId, activeServerId,
}; };
return ( return (
<RealtimeContext.Provider value={value}> <VoiceContext.Provider value={value}>
{children} {children}
</RealtimeContext.Provider> </VoiceContext.Provider>
); );
} }
packages/app/src/hooks/use-audio-player.web.ts
+1 -1
View File
@@ -7,7 +7,7 @@ export interface AudioPlayerOptions {
/** /**
* Web shim for the native two-way audio player. * Web shim for the native two-way audio player.
* We currently don't support realtime voice playback on web. * We currently don't support Voice mode playback on web.
*/ */
export function useAudioPlayer(_options?: AudioPlayerOptions) { export function useAudioPlayer(_options?: AudioPlayerOptions) {
return useMemo( return useMemo(
packages/server/agent-prompt.md
+9
View File
@@ -6,6 +6,15 @@
You are a **voice-controlled** assistant. The user speaks to you via phone and hears your responses via TTS. You are a **voice-controlled** assistant. The user speaks to you via phone and hears your responses via TTS.
### Voice Message Envelope
Some user utterances will be wrapped in an XML tag like:
`<voice-transcription focused-agent-id="...">...</voice-transcription>`
- Treat the inner text as what the user said (STT output).
- `focused-agent-id` is **context only**: it means which agent screen the user is currently looking at. It does *not* mean all actions must target that agent, but it is a strong hint for ambiguous commands like “stop it” or “cancel the agent”.
**Critical constraints:** **Critical constraints:**
- User typically codes from their **phone** using voice - User typically codes from their **phone** using voice
packages/server/src/client/daemon-client-v2.ts
+2 -2
View File
@@ -1132,12 +1132,12 @@ export class DaemonClientV2 {
this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId }); this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId });
} }
async sendRealtimeAudioChunk( async sendVoiceAudioChunk(
audio: string, audio: string,
format: string, format: string,
isLast: boolean isLast: boolean
): Promise<void> { ): Promise<void> {
this.sendSessionMessage({ type: "realtime_audio_chunk", audio, format, isLast }); this.sendSessionMessage({ type: "voice_audio_chunk", audio, format, isLast });
} }
startDictationStream(dictationId: string, format: string): Promise<void> { startDictationStream(dictationId: string, format: string): Promise<void> {
packages/server/src/server/agent/tts-manager.ts
+2 -2
View File
@@ -32,7 +32,7 @@ export class TTSManager {
text: string, text: string,
emitMessage: (msg: SessionOutboundMessage) => void, emitMessage: (msg: SessionOutboundMessage) => void,
abortSignal: AbortSignal, abortSignal: AbortSignal,
isRealtimeMode: boolean isVoiceMode: boolean
): Promise<void> { ): Promise<void> {
if (!this.tts) { if (!this.tts) {
throw new Error("TTS not configured"); throw new Error("TTS not configured");
@@ -117,7 +117,7 @@ export class TTSManager {
isLastChunk: next.done, isLastChunk: next.done,
audio: chunkBuffer.toString("base64"), audio: chunkBuffer.toString("base64"),
format, format,
isRealtimeMode, isVoiceMode,
}, },
}); });
packages/server/src/server/agent/wait-for-agent-tracker.ts
+1 -1
View File
@@ -4,7 +4,7 @@ export type WaitForAgentCanceler = (agentId: string, reason?: string) => boolean
/** /**
* Tracks long-running wait_for_agent tool calls so they can be cancelled * Tracks long-running wait_for_agent tool calls so they can be cancelled
* explicitly (e.g., when a realtime barge-in aborts the current turn). * explicitly (e.g., when a voice barge-in aborts the current turn).
*/ */
export class WaitForAgentTracker { export class WaitForAgentTracker {
private waiters = new Map<string, Set<(reason?: string) => void>>(); private waiters = new Map<string, Set<(reason?: string) => void>>();
packages/server/src/server/daemon-client-v2.e2e.test.ts
+3 -3
View File
@@ -544,7 +544,7 @@ describe("daemon client v2 E2E", () => {
); );
test( test(
"realtime mode buffers audio until isLast and emits transcription_result", "voice mode buffers audio until isLast and emits transcription_result",
async () => { async () => {
requireEnv("OPENAI_API_KEY"); requireEnv("OPENAI_API_KEY");
@@ -613,7 +613,7 @@ describe("daemon client v2 E2E", () => {
const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16 const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
const firstChunk = pcm16.subarray(0, Math.min(chunkBytes, pcm16.length)); const firstChunk = pcm16.subarray(0, Math.min(chunkBytes, pcm16.length));
await ctx.client.sendRealtimeAudioChunk(firstChunk.toString("base64"), format, false); await ctx.client.sendVoiceAudioChunk(firstChunk.toString("base64"), format, false);
await earlyTranscription await earlyTranscription
.then(() => { .then(() => {
throw new Error("Expected no transcription_result before isLast=true"); throw new Error("Expected no transcription_result before isLast=true");
@@ -623,7 +623,7 @@ describe("daemon client v2 E2E", () => {
for (let offset = chunkBytes; offset < pcm16.length; offset += chunkBytes) { for (let offset = chunkBytes; offset < pcm16.length; offset += chunkBytes) {
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
const isLast = offset + chunkBytes >= pcm16.length; const isLast = offset + chunkBytes >= pcm16.length;
await ctx.client.sendRealtimeAudioChunk(chunk.toString("base64"), format, isLast); await ctx.client.sendVoiceAudioChunk(chunk.toString("base64"), format, isLast);
} }
const outcome = await Promise.race([ const outcome = await Promise.race([
packages/server/src/server/session.ts
+52 -25
View File
@@ -260,8 +260,8 @@ export class Session {
private processingPhase: ProcessingPhase = "idle"; private processingPhase: ProcessingPhase = "idle";
private currentStreamPromise: Promise<void> | null = null; private currentStreamPromise: Promise<void> | null = null;
// Realtime mode state // Voice mode state
private isRealtimeMode = false; private isVoiceMode = false;
private speechInProgress = false; private speechInProgress = false;
private voiceConversationId: string | null = null; private voiceConversationId: string | null = null;
@@ -369,6 +369,28 @@ export class Session {
this.sessionLogger.info("Session created"); this.sessionLogger.info("Session created");
} }
private escapeXmlText(value: string): string {
return value
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
private escapeXmlAttribute(value: string): string {
return this.escapeXmlText(value)
.replace(/"/g, "&quot;")
.replace(/'/g, "&apos;");
}
private formatVoiceTranscriptionForLLM(text: string): string {
const trimmed = text.trim();
const focusedAgentId = this.clientActivity?.focusedAgentId ?? null;
const focusedAttr = focusedAgentId
? ` focused-agent-id="${this.escapeXmlAttribute(focusedAgentId)}"`
: "";
return `<voice-transcription${focusedAttr}>${this.escapeXmlText(trimmed)}</voice-transcription>`;
}
/** /**
* Get the client's current activity state * Get the client's current activity state
*/ */
@@ -735,7 +757,7 @@ export class Session {
await this.handleUserText(msg.text); await this.handleUserText(msg.text);
break; break;
case "realtime_audio_chunk": case "voice_audio_chunk":
await this.handleAudioChunk(msg); await this.handleAudioChunk(msg);
break; break;
@@ -1214,7 +1236,7 @@ export class Session {
} }
/** /**
* Handle realtime mode toggle * Handle voice mode toggle
*/ */
private async handleSetVoiceConversation( private async handleSetVoiceConversation(
enabled: boolean, enabled: boolean,
@@ -1226,7 +1248,7 @@ export class Session {
return; return;
} }
this.isRealtimeMode = true; this.isVoiceMode = true;
this.voiceConversationId = voiceConversationId; this.voiceConversationId = voiceConversationId;
const loaded = await this.voiceConversationStore.load( const loaded = await this.voiceConversationStore.load(
@@ -1242,7 +1264,7 @@ export class Session {
return; return;
} }
this.isRealtimeMode = false; this.isVoiceMode = false;
const idToPersist = this.voiceConversationId; const idToPersist = this.voiceConversationId;
if (idToPersist) { if (idToPersist) {
try { try {
@@ -3875,8 +3897,8 @@ export class Session {
// Add to conversation // Add to conversation
this.messages.push({ role: "user", content: text }); this.messages.push({ role: "user", content: text });
// Process through LLM (TTS enabled in realtime mode for voice conversations) // Process through LLM (TTS enabled in voice mode for voice conversations)
this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode); this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
await this.currentStreamPromise; await this.currentStreamPromise;
} }
@@ -3884,9 +3906,9 @@ export class Session {
* Handle audio chunk for buffering and transcription * Handle audio chunk for buffering and transcription
*/ */
private async handleAudioChunk( private async handleAudioChunk(
msg: Extract<SessionInboundMessage, { type: "realtime_audio_chunk" }> msg: Extract<SessionInboundMessage, { type: "voice_audio_chunk" }>
): Promise<void> { ): Promise<void> {
await this.handleRealtimeSpeechStart(); await this.handleVoiceSpeechStart();
const chunkBuffer = Buffer.from(msg.audio, "base64"); const chunkBuffer = Buffer.from(msg.audio, "base64");
const chunkFormat = msg.format || "audio/wav"; const chunkFormat = msg.format || "audio/wav";
@@ -3932,19 +3954,19 @@ export class Session {
`Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})` `Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})`
); );
// In realtime mode, only process audio when the user has finished speaking (isLast = true) // In voice mode, only process audio when the user has finished speaking (isLast = true)
// This prevents partial transcriptions from being sent to the LLM // This prevents partial transcriptions from being sent to the LLM
if (this.isRealtimeMode) { if (this.isVoiceMode) {
if (!msg.isLast) { if (!msg.isLast) {
this.sessionLogger.debug("Realtime mode: buffering audio, waiting for speech end"); this.sessionLogger.debug("Voice mode: buffering audio, waiting for speech end");
return; return;
} }
this.sessionLogger.debug("Realtime mode: speech ended, processing complete audio"); this.sessionLogger.debug("Voice mode: speech ended, processing complete audio");
} }
// In non-realtime mode, use streaming threshold to process chunks // In non-voice mode, use streaming threshold to process chunks
const reachedStreamingThreshold = const reachedStreamingThreshold =
!this.isRealtimeMode && !this.isVoiceMode &&
this.audioBuffer.isPCM && this.audioBuffer.isPCM &&
this.audioBuffer.totalPCMBytes >= MIN_STREAMING_SEGMENT_BYTES; this.audioBuffer.totalPCMBytes >= MIN_STREAMING_SEGMENT_BYTES;
@@ -4073,7 +4095,7 @@ export class Session {
const requestId = uuidv4(); const requestId = uuidv4();
const result = await this.sttManager.transcribe(audio, format, { const result = await this.sttManager.transcribe(audio, format, {
requestId, requestId,
label: this.isRealtimeMode ? "realtime" : "buffered", label: this.isVoiceMode ? "voice" : "buffered",
}); });
const transcriptText = result.text.trim(); const transcriptText = result.text.trim();
@@ -4143,12 +4165,17 @@ export class Session {
}); });
// Add to conversation // Add to conversation
this.messages.push({ role: "user", content: result.text }); this.messages.push({
role: "user",
content: this.isVoiceMode
? this.formatVoiceTranscriptionForLLM(result.text)
: result.text,
});
// Set phase to LLM and process (TTS enabled in realtime mode for voice conversations) // Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
this.clearSpeechInProgress("transcription complete"); this.clearSpeechInProgress("transcription complete");
this.setPhase("llm"); this.setPhase("llm");
this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode); this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
await this.currentStreamPromise; await this.currentStreamPromise;
this.setPhase("idle"); this.setPhase("idle");
} catch (error: any) { } catch (error: any) {
@@ -4180,7 +4207,7 @@ export class Session {
if (textBuffer.length > 0) { if (textBuffer.length > 0) {
// TTS handling (capture mode at generation time for drift protection) // TTS handling (capture mode at generation time for drift protection)
if (enableTTS && !this.speechInProgress) { if (enableTTS && !this.speechInProgress) {
const modeAtGeneration = this.isRealtimeMode; const modeAtGeneration = this.isVoiceMode;
pendingTTS = this.ttsManager.generateAndWaitForPlayback( pendingTTS = this.ttsManager.generateAndWaitForPlayback(
textBuffer, textBuffer,
(msg) => this.emit(msg), (msg) => this.emit(msg),
@@ -4572,7 +4599,7 @@ export class Session {
/** /**
* Mark speech detection start and abort any active playback/LLM * Mark speech detection start and abort any active playback/LLM
*/ */
private async handleRealtimeSpeechStart(): Promise<void> { private async handleVoiceSpeechStart(): Promise<void> {
if (this.speechInProgress) { if (this.speechInProgress) {
return; return;
} }
@@ -4582,13 +4609,13 @@ export class Session {
const hadActiveStream = Boolean(this.currentStreamPromise); const hadActiveStream = Boolean(this.currentStreamPromise);
this.speechInProgress = true; this.speechInProgress = true;
this.sessionLogger.debug("Realtime speech chunk detected – aborting playback and LLM"); this.sessionLogger.debug("Voice speech detected – aborting playback and LLM");
this.ttsManager.cancelPendingPlaybacks("realtime speech detected"); this.ttsManager.cancelPendingPlaybacks("voice speech detected");
if (this.pendingAudioSegments.length > 0) { if (this.pendingAudioSegments.length > 0) {
this.sessionLogger.debug( this.sessionLogger.debug(
{ segmentCount: this.pendingAudioSegments.length }, { segmentCount: this.pendingAudioSegments.length },
`Dropping ${this.pendingAudioSegments.length} buffered audio segment(s) due to realtime speech` `Dropping ${this.pendingAudioSegments.length} buffered audio segment(s) due to voice speech`
); );
this.pendingAudioSegments = []; this.pendingAudioSegments = [];
} }
packages/server/src/shared/messages.ts
+5 -5
View File
@@ -288,8 +288,8 @@ export const UserTextMessageSchema = z.object({
text: z.string(), text: z.string(),
}); });
export const RealtimeAudioChunkMessageSchema = z.object({ export const VoiceAudioChunkMessageSchema = z.object({
type: z.literal("realtime_audio_chunk"), type: z.literal("voice_audio_chunk"),
audio: z.string(), // base64 encoded audio: z.string(), // base64 encoded
format: z.string(), format: z.string(),
isLast: z.boolean(), isLast: z.boolean(),
@@ -804,7 +804,7 @@ export const KillTerminalRequestSchema = z.object({
export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
UserTextMessageSchema, UserTextMessageSchema,
RealtimeAudioChunkMessageSchema, VoiceAudioChunkMessageSchema,
AbortRequestMessageSchema, AbortRequestMessageSchema,
AudioPlayedMessageSchema, AudioPlayedMessageSchema,
FetchAgentsRequestMessageSchema, FetchAgentsRequestMessageSchema,
@@ -900,7 +900,7 @@ export const AudioOutputMessageSchema = z.object({
audio: z.string(), // base64 encoded audio: z.string(), // base64 encoded
format: z.string(), format: z.string(),
id: z.string(), id: z.string(),
isRealtimeMode: z.boolean(), // Mode when audio was generated (for drift protection) isVoiceMode: z.boolean(), // Mode when audio was generated (for drift protection)
groupId: z.string().optional(), // Logical utterance id groupId: z.string().optional(), // Logical utterance id
chunkIndex: z.number().int().nonnegative().optional(), chunkIndex: z.number().int().nonnegative().optional(),
isLastChunk: z.boolean().optional(), isLastChunk: z.boolean().optional(),
@@ -1643,7 +1643,7 @@ export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
// Type exports for inbound message types // Type exports for inbound message types
export type UserTextMessage = z.infer<typeof UserTextMessageSchema>; export type UserTextMessage = z.infer<typeof UserTextMessageSchema>;
export type RealtimeAudioChunkMessage = z.infer<typeof RealtimeAudioChunkMessageSchema>; export type VoiceAudioChunkMessage = z.infer<typeof VoiceAudioChunkMessageSchema>;
export type FetchAgentsRequestMessage = z.infer<typeof FetchAgentsRequestMessageSchema>; export type FetchAgentsRequestMessage = z.infer<typeof FetchAgentsRequestMessageSchema>;
export type FetchAgentRequestMessage = z.infer<typeof FetchAgentRequestMessageSchema>; export type FetchAgentRequestMessage = z.infer<typeof FetchAgentRequestMessageSchema>;
export type SendAgentMessageRequest = z.infer<typeof SendAgentMessageRequestSchema>; export type SendAgentMessageRequest = z.infer<typeof SendAgentMessageRequestSchema>;