Update files

This commit is contained in:
Mohamed Boudra
2026-02-09 21:26:12 +07:00
parent ab23d5d9b1
commit 7a21a5cce1
7 changed files with 465 additions and 167 deletions
@@ -31,6 +31,8 @@ import { encodeImages } from "@/utils/encode-images";
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store"; import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
import { focusWithRetries } from "@/utils/web-focus"; import { focusWithRetries } from "@/utils/web-focus";
import { useVoiceOptional } from "@/contexts/voice-context"; import { useVoiceOptional } from "@/contexts/voice-context";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { Shortcut } from "@/components/ui/shortcut";
type QueuedMessage = { type QueuedMessage = {
id: string; id: string;
@@ -512,10 +514,23 @@ export function AgentInputArea({
}, },
[setUserInput] [setUserInput]
); );
const hasSendableContent = userInput.trim().length > 0 || selectedImages.length > 0;
// Handle keyboard navigation for command autocomplete // Handle keyboard navigation for command autocomplete
const handleCommandKeyPress = useCallback( const handleCommandKeyPress = useCallback(
(event: { key: string; preventDefault: () => void }) => { (event: { key: string; preventDefault: () => void }) => {
if (
event.key === "Escape" &&
isAgentRunning &&
!hasSendableContent &&
!isCancellingAgent &&
isConnected
) {
event.preventDefault();
handleCancelAgent();
return true;
}
if (!showCommandAutocomplete || filteredCommands.length === 0) { if (!showCommandAutocomplete || filteredCommands.length === 0) {
return false; return false;
} }
@@ -558,54 +573,72 @@ export function AgentInputArea({
filteredCommands, filteredCommands,
commandSelectedIndex, commandSelectedIndex,
handleCommandSelect, handleCommandSelect,
hasSendableContent,
isAgentRunning,
isCancellingAgent,
isConnected,
setUserInput, setUserInput,
] ]
); );
const hasSendableContent = userInput.trim().length > 0 || selectedImages.length > 0;
const cancelButton = isAgentRunning && !hasSendableContent ? ( const cancelButton = isAgentRunning && !hasSendableContent ? (
<Pressable <Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
onPress={handleCancelAgent} <TooltipTrigger
disabled={!isConnected || isCancellingAgent} onPress={handleCancelAgent}
accessibilityLabel={isCancellingAgent ? "Canceling agent" : "Stop agent"} disabled={!isConnected || isCancellingAgent}
accessibilityRole="button" accessibilityLabel={isCancellingAgent ? "Canceling agent" : "Stop agent"}
style={[ accessibilityRole="button"
styles.cancelButton as any, style={[
(!isConnected || isCancellingAgent styles.cancelButton as any,
? styles.buttonDisabled (!isConnected || isCancellingAgent
: undefined) as any, ? styles.buttonDisabled
]} : undefined) as any,
> ]}
{isCancellingAgent ? ( >
<ActivityIndicator size="small" color="white" /> {isCancellingAgent ? (
) : ( <ActivityIndicator size="small" color="white" />
<Square size={18} color="white" fill="white" /> ) : (
)} <Square size={18} color="white" fill="white" />
</Pressable> )}
</TooltipTrigger>
<TooltipContent side="top" align="center" offset={8}>
<View style={styles.tooltipRow}>
<Text style={styles.tooltipText}>Interrupt</Text>
<Shortcut keys={["Esc"]} style={styles.tooltipShortcut} />
</View>
</TooltipContent>
</Tooltip>
) : null; ) : null;
const rightContent = ( const rightContent = (
<View style={styles.rightControls}> <View style={styles.rightControls}>
{!isVoiceModeForAgent ? ( {!isVoiceModeForAgent ? (
<Pressable <Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
onPress={handleToggleRealtimeVoice} <TooltipTrigger
disabled={!isConnected || voice?.isVoiceSwitching} onPress={handleToggleRealtimeVoice}
accessibilityLabel="Enable realtime voice mode" disabled={!isConnected || voice?.isVoiceSwitching}
accessibilityRole="button" accessibilityLabel="Enable Voice mode"
style={[ accessibilityRole="button"
styles.realtimeVoiceButton as any, style={[
(!isConnected || voice?.isVoiceSwitching styles.realtimeVoiceButton as any,
? styles.buttonDisabled (!isConnected || voice?.isVoiceSwitching
: undefined) as any, ? styles.buttonDisabled
]} : undefined) as any,
> ]}
{voice?.isVoiceSwitching ? ( >
<ActivityIndicator size="small" color="white" /> {voice?.isVoiceSwitching ? (
) : ( <ActivityIndicator size="small" color="white" />
<AudioLines size={18} color={theme.colors.foreground} /> ) : (
)} <AudioLines size={18} color={theme.colors.foreground} />
</Pressable> )}
</TooltipTrigger>
<TooltipContent side="top" align="center" offset={8}>
<View style={styles.tooltipRow}>
<Text style={styles.tooltipText}>Voice mode</Text>
<Shortcut keys={["mod", "shift", "D"]} style={styles.tooltipShortcut} />
</View>
</TooltipContent>
</Tooltip>
) : null} ) : null}
{cancelButton} {cancelButton}
</View> </View>
@@ -647,7 +680,7 @@ export function AgentInputArea({
onPress={() => handleSendQueuedNow(item.id)} onPress={() => handleSendQueuedNow(item.id)}
style={[styles.queueActionButton, styles.queueSendButton]} style={[styles.queueActionButton, styles.queueSendButton]}
> >
<ArrowUp size={14} color={theme.colors.background} /> <ArrowUp size={14} color="white" />
</Pressable> </Pressable>
</View> </View>
</View> </View>
@@ -752,6 +785,19 @@ const styles = StyleSheet.create(((theme: Theme) => ({
backgroundColor: theme.colors.palette.green[600], backgroundColor: theme.colors.palette.green[600],
borderColor: theme.colors.palette.green[800], borderColor: theme.colors.palette.green[800],
}, },
tooltipRow: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[2],
},
tooltipText: {
fontSize: theme.fontSize.sm,
color: theme.colors.popoverForeground,
},
tooltipShortcut: {
backgroundColor: theme.colors.surface3,
borderColor: theme.colors.borderAccent,
},
buttonDisabled: { buttonDisabled: {
opacity: 0.5, opacity: 0.5,
}, },
@@ -765,7 +811,7 @@ const styles = StyleSheet.create(((theme: Theme) => ({
justifyContent: "space-between", justifyContent: "space-between",
paddingHorizontal: theme.spacing[3], paddingHorizontal: theme.spacing[3],
paddingVertical: theme.spacing[2], paddingVertical: theme.spacing[2],
backgroundColor: theme.colors.surface2, backgroundColor: theme.colors.surface1,
borderRadius: theme.borderRadius.lg, borderRadius: theme.borderRadius.lg,
borderWidth: theme.borderWidth[1], borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border, borderColor: theme.colors.border,
@@ -790,7 +836,7 @@ const styles = StyleSheet.create(((theme: Theme) => ({
backgroundColor: theme.colors.surface2, backgroundColor: theme.colors.surface2,
}, },
queueSendButton: { queueSendButton: {
backgroundColor: theme.colors.palette.blue[600], backgroundColor: theme.colors.accent,
}, },
sendErrorText: { sendErrorText: {
color: theme.colors.palette.red[500], color: theme.colors.palette.red[500],
@@ -377,14 +377,14 @@ const styles = StyleSheet.create((theme) => ({
modeBadge: { modeBadge: {
flexDirection: "row", flexDirection: "row",
alignItems: "center", alignItems: "center",
backgroundColor: theme.colors.surface2, backgroundColor: "transparent",
gap: theme.spacing[1], gap: theme.spacing[1],
paddingHorizontal: theme.spacing[2], paddingHorizontal: theme.spacing[2],
paddingVertical: theme.spacing[1], paddingVertical: theme.spacing[1],
borderRadius: theme.borderRadius["2xl"], borderRadius: theme.borderRadius["2xl"],
}, },
modeBadgeHovered: { modeBadgeHovered: {
backgroundColor: theme.colors.surface1, backgroundColor: theme.colors.surface2,
}, },
modeBadgePressed: { modeBadgePressed: {
backgroundColor: theme.colors.surface0, backgroundColor: theme.colors.surface0,
+241 -80
View File
@@ -1,5 +1,6 @@
import { import {
View, View,
Text,
TextInput, TextInput,
Pressable, Pressable,
ActivityIndicator, ActivityIndicator,
@@ -31,6 +32,8 @@ import { RealtimeVoiceOverlay } from "./realtime-voice-overlay";
import type { DaemonClient } from "@server/client/daemon-client"; import type { DaemonClient } from "@server/client/daemon-client";
import { usePanelStore } from "@/stores/panel-store"; import { usePanelStore } from "@/stores/panel-store";
import { useVoiceOptional } from "@/contexts/voice-context"; import { useVoiceOptional } from "@/contexts/voice-context";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { Shortcut } from "@/components/ui/shortcut";
export interface ImageAttachment { export interface ImageAttachment {
uri: string; uri: string;
@@ -217,6 +220,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
const socketConnected = client?.isConnected ?? false; const socketConnected = client?.isConnected ?? false;
return socketConnected; return socketConnected;
}, [client]); }, [client]);
const isConnected = client?.isConnected ?? false;
const { const {
isRecording: isDictating, isRecording: isDictating,
@@ -263,9 +267,39 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
sendAfterTranscriptRef.current = false; sendAfterTranscriptRef.current = false;
}, [dictationStatus, isDictating, isDictationProcessing]); }, [dictationStatus, isDictating, isDictationProcessing]);
// Cmd+D to start/submit dictation, Escape to cancel // Cmd+D to start/submit dictation, Cmd+Shift+D toggles realtime voice, Escape cancels dictation
useEffect(() => { useEffect(() => {
if (!IS_WEB) return; if (!IS_WEB) return;
const toggleRealtimeVoice = () => {
if (!voice || !voiceServerId || !voiceAgentId || !isConnected || disabled) {
return;
}
if (voice.isVoiceSwitching) {
return;
}
if (voice.isVoiceModeForAgent(voiceServerId, voiceAgentId)) {
const tasks: Promise<unknown>[] = [];
if (isAgentRunning && client) {
tasks.push(client.cancelAgent(voiceAgentId));
}
tasks.push(voice.stopVoice());
void Promise.allSettled(tasks).then((results) => {
results.forEach((result) => {
if (result.status === "rejected") {
console.error(
"[MessageInput] Failed to stop realtime voice",
result.reason
);
}
});
});
return;
}
void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
console.error("[MessageInput] Failed to start realtime voice", error);
});
};
const resolveNativeInput = (): unknown => { const resolveNativeInput = (): unknown => {
const current = textInputRef.current as any; const current = textInputRef.current as any;
if (!current) return null; if (!current) return null;
@@ -280,12 +314,32 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
if (!isScreenFocused && !isInputFocusedRef.current && !isFromInput) { if (!isScreenFocused && !isInputFocusedRef.current && !isFromInput) {
return; return;
} }
const isMod = event.metaKey || event.ctrlKey;
const isKeyD = event.code === "KeyD" || event.key.toLowerCase() === "d";
if (isMod && event.shiftKey && isKeyD && !event.repeat) {
event.preventDefault();
toggleRealtimeVoice();
return;
}
if (
isRealtimeVoiceForCurrentAgent &&
!event.metaKey &&
!event.ctrlKey &&
!event.altKey &&
!event.shiftKey &&
(event.code === "Space" || event.key === " ") &&
!event.repeat
) {
event.preventDefault();
voice?.toggleMute();
return;
}
const dictating = isDictatingRef.current; const dictating = isDictatingRef.current;
// Cmd+D: start dictation or submit if already dictating // Cmd+D: start dictation or submit if already dictating
if ( if (isMod && isKeyD) {
(event.metaKey || event.ctrlKey) &&
(event.code === "KeyD" || event.key.toLowerCase() === "d")
) {
event.preventDefault(); event.preventDefault();
if (dictating) { if (dictating) {
sendAfterTranscriptRef.current = true; sendAfterTranscriptRef.current = true;
@@ -303,7 +357,20 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
} }
window.addEventListener("keydown", handleKeyDown); window.addEventListener("keydown", handleKeyDown);
return () => window.removeEventListener("keydown", handleKeyDown); return () => window.removeEventListener("keydown", handleKeyDown);
}, [cancelDictation, confirmDictation, isScreenFocused, startDictation]); }, [
cancelDictation,
client,
confirmDictation,
disabled,
isAgentRunning,
isConnected,
isRealtimeVoiceForCurrentAgent,
isScreenFocused,
startDictation,
voiceAgentId,
voiceServerId,
voice,
]);
// Animate overlay // Animate overlay
useEffect(() => { useEffect(() => {
@@ -381,6 +448,29 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
}); });
}, [client, isAgentRunning, isRealtimeVoiceForCurrentAgent, voice, voiceAgentId]); }, [client, isAgentRunning, isRealtimeVoiceForCurrentAgent, voice, voiceAgentId]);
const handleToggleRealtimeVoiceShortcut = useCallback(() => {
if (!voice || !voiceServerId || !voiceAgentId || !isConnected || disabled) {
return;
}
if (voice.isVoiceSwitching) {
return;
}
if (voice.isVoiceModeForAgent(voiceServerId, voiceAgentId)) {
void handleStopRealtimeVoice();
return;
}
void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
console.error("[MessageInput] Failed to start realtime voice", error);
});
}, [
disabled,
handleStopRealtimeVoice,
isConnected,
voice,
voiceAgentId,
voiceServerId,
]);
const handleSendMessage = useCallback(() => { const handleSendMessage = useCallback(() => {
const trimmed = value.trim(); const trimmed = value.trim();
if (!trimmed && images.length === 0) return; if (!trimmed && images.length === 0) return;
@@ -514,6 +604,13 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
return; return;
} }
// Cmd+Shift+D or Ctrl+Shift+D: toggle realtime voice mode
if ((metaKey || ctrlKey) && shiftKey && key === "d") {
event.preventDefault();
handleToggleRealtimeVoiceShortcut();
return;
}
// Cmd+D or Ctrl+D: start dictation or submit if already dictating // Cmd+D or Ctrl+D: start dictation or submit if already dictating
if ((metaKey || ctrlKey) && key === "d") { if ((metaKey || ctrlKey) && key === "d") {
event.preventDefault(); event.preventDefault();
@@ -533,6 +630,18 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
return; return;
} }
if (
isRealtimeVoiceForCurrentAgent &&
!metaKey &&
!ctrlKey &&
!shiftKey &&
event.nativeEvent.key === " "
) {
event.preventDefault();
voice?.toggleMute();
return;
}
if (event.nativeEvent.key !== "Enter") return; if (event.nativeEvent.key !== "Enter") return;
// Shift+Enter: add newline (default behavior, don't intercept) // Shift+Enter: add newline (default behavior, don't intercept)
@@ -555,7 +664,6 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
const hasImages = images.length > 0; const hasImages = images.length > 0;
const hasSendableContent = value.trim().length > 0 || hasImages; const hasSendableContent = value.trim().length > 0 || hasImages;
const shouldShowSendButton = hasSendableContent || isSubmitLoading; const shouldShowSendButton = hasSendableContent || isSubmitLoading;
const isConnected = client?.isConnected ?? false;
return ( return (
<View style={styles.container}> <View style={styles.container}>
@@ -636,90 +744,130 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
{/* Left: attachment button + leftContent slot */} {/* Left: attachment button + leftContent slot */}
<View style={styles.leftButtonGroup}> <View style={styles.leftButtonGroup}>
{onPickImages && ( {onPickImages && (
<Pressable <Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
onPress={onPickImages} <TooltipTrigger
disabled={!isConnected || disabled} onPress={onPickImages}
style={[ disabled={!isConnected || disabled}
styles.attachButton, accessibilityLabel="Attach images"
(!isConnected || disabled) && styles.buttonDisabled, accessibilityRole="button"
]} style={[
> styles.attachButton,
<Paperclip size={20} color={theme.colors.foreground} /> (!isConnected || disabled) && styles.buttonDisabled,
</Pressable> ]}
>
<Paperclip size={20} color={theme.colors.foreground} />
</TooltipTrigger>
<TooltipContent side="top" align="center" offset={8}>
<Text style={styles.tooltipText}>Attach images</Text>
</TooltipContent>
</Tooltip>
)} )}
{leftContent} {leftContent}
</View> </View>
{/* Right: voice button, contextual button (realtime/send/cancel) */} {/* Right: voice button, contextual button (realtime/send/cancel) */}
<View style={styles.rightButtonGroup}> <View style={styles.rightButtonGroup}>
<Pressable <Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
onPress={handleVoicePress} <TooltipTrigger
disabled={!isConnected || disabled} onPress={handleVoicePress}
accessibilityRole="button" disabled={!isConnected || disabled}
accessibilityLabel={ accessibilityRole="button"
isRealtimeVoiceForCurrentAgent accessibilityLabel={
? voice?.isMuted isRealtimeVoiceForCurrentAgent
? "Unmute realtime voice" ? voice?.isMuted
: "Mute realtime voice" ? "Unmute Voice mode"
: isDictating : "Mute Voice mode"
? "Stop dictation" : isDictating
: "Start dictation" ? "Stop dictation"
} : "Start dictation"
style={[ }
styles.voiceButton, style={[
(!isConnected || disabled) && styles.buttonDisabled, styles.voiceButton,
isDictating && styles.voiceButtonRecording, (!isConnected || disabled) && styles.buttonDisabled,
]} isDictating && styles.voiceButtonRecording,
> ]}
{isDictating ? ( >
<Square size={14} color="white" fill="white" /> {isDictating ? (
) : isRealtimeVoiceForCurrentAgent && voice?.isMuted ? ( <Square size={14} color="white" fill="white" />
<MicOff size={20} color={theme.colors.foreground} /> ) : isRealtimeVoiceForCurrentAgent && voice?.isMuted ? (
) : ( <MicOff size={20} color={theme.colors.foreground} />
<Mic size={20} color={theme.colors.foreground} /> ) : (
)} <Mic size={20} color={theme.colors.foreground} />
</Pressable> )}
</TooltipTrigger>
<TooltipContent side="top" align="center" offset={8}>
<View style={styles.tooltipRow}>
<Text style={styles.tooltipText}>
{isRealtimeVoiceForCurrentAgent
? voice?.isMuted
? "Unmute voice"
: "Mute voice"
: "Dictation"}
</Text>
<Shortcut
keys={isRealtimeVoiceForCurrentAgent ? ["Space"] : ["mod", "D"]}
style={styles.tooltipShortcut}
/>
</View>
</TooltipContent>
</Tooltip>
{rightContent} {rightContent}
{shouldShowSendButton && isAgentRunning && onQueue && ( {shouldShowSendButton && isAgentRunning && onQueue && (
<Pressable <Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
onPress={handleQueueMessage} <TooltipTrigger
disabled={!isConnected || disabled} onPress={handleQueueMessage}
accessibilityLabel="Queue message" disabled={!isConnected || disabled}
accessibilityRole="button" accessibilityLabel="Queue message"
style={[ accessibilityRole="button"
styles.queueButton, style={[
(!isConnected || disabled) && styles.buttonDisabled, styles.queueButton,
]} (!isConnected || disabled) && styles.buttonDisabled,
> ]}
<Plus size={20} color="white" /> >
</Pressable> <Plus size={20} color="white" />
</TooltipTrigger>
<TooltipContent side="top" align="center" offset={8}>
<View style={styles.tooltipRow}>
<Text style={styles.tooltipText}>Queue</Text>
<Shortcut keys={["mod", "Enter"]} style={styles.tooltipShortcut} />
</View>
</TooltipContent>
</Tooltip>
)} )}
{shouldShowSendButton && ( {shouldShowSendButton && (
<Pressable <Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
onPress={handleSendMessage} <TooltipTrigger
disabled={ onPress={handleSendMessage}
!isConnected || disabled={
isSubmitDisabled || !isConnected ||
isSubmitLoading ||
disabled
}
accessibilityLabel={isAgentRunning ? "Send and interrupt" : "Send message"}
accessibilityRole="button"
style={[
styles.sendButton,
(!isConnected ||
isSubmitDisabled || isSubmitDisabled ||
isSubmitLoading || isSubmitLoading ||
disabled) && disabled
styles.buttonDisabled, }
]} accessibilityLabel={isAgentRunning ? "Send and interrupt" : "Send message"}
> accessibilityRole="button"
{isSubmitLoading ? ( style={[
<ActivityIndicator size="small" color="white" /> styles.sendButton,
) : ( (!isConnected ||
<ArrowUp size={20} color="white" /> isSubmitDisabled ||
)} isSubmitLoading ||
</Pressable> disabled) &&
styles.buttonDisabled,
]}
>
{isSubmitLoading ? (
<ActivityIndicator size="small" color="white" />
) : (
<ArrowUp size={20} color="white" />
)}
</TooltipTrigger>
<TooltipContent side="top" align="center" offset={8}>
<View style={styles.tooltipRow}>
<Text style={styles.tooltipText}>Send</Text>
<Shortcut keys={["Enter"]} style={styles.tooltipShortcut} />
</View>
</TooltipContent>
</Tooltip>
)} )}
</View> </View>
</View> </View>
@@ -775,7 +923,7 @@ const styles = StyleSheet.create(((theme: any) => ({
inputWrapper: { inputWrapper: {
flexDirection: "column", flexDirection: "column",
gap: theme.spacing[3], gap: theme.spacing[3],
backgroundColor: theme.colors.surface2, backgroundColor: theme.colors.surface1,
borderWidth: theme.borderWidth[1], borderWidth: theme.borderWidth[1],
borderColor: theme.colors.borderAccent, borderColor: theme.colors.borderAccent,
borderRadius: theme.borderRadius["2xl"], borderRadius: theme.borderRadius["2xl"],
@@ -898,6 +1046,19 @@ const styles = StyleSheet.create(((theme: any) => ({
alignItems: "center", alignItems: "center",
justifyContent: "center", justifyContent: "center",
}, },
tooltipRow: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[2],
},
tooltipText: {
fontSize: theme.fontSize.sm,
color: theme.colors.popoverForeground,
},
tooltipShortcut: {
backgroundColor: theme.colors.surface3,
borderColor: theme.colors.borderAccent,
},
buttonDisabled: { buttonDisabled: {
opacity: 0.5, opacity: 0.5,
}, },
+2 -2
View File
@@ -463,9 +463,9 @@ const styles = StyleSheet.create((theme) => ({
content: { content: {
paddingVertical: theme.spacing[1], paddingVertical: theme.spacing[1],
paddingHorizontal: theme.spacing[2], paddingHorizontal: theme.spacing[2],
borderRadius: theme.borderRadius.lg, borderRadius: theme.borderRadius.xl,
backgroundColor: theme.colors.popover, backgroundColor: theme.colors.popover,
borderWidth: theme.borderWidth[1], borderWidth: theme.borderWidth[2],
borderColor: theme.colors.border, borderColor: theme.colors.border,
shadowColor: "#000", shadowColor: "#000",
shadowOpacity: 0.2, shadowOpacity: 0.2,
+8 -41
View File
@@ -68,6 +68,11 @@ import type {
} from "./agent/agent-sdk-types.js"; } from "./agent/agent-sdk-types.js";
import { AgentStorage, type StoredAgentRecord } from "./agent/agent-storage.js"; import { AgentStorage, type StoredAgentRecord } from "./agent/agent-storage.js";
import { isValidAgentProvider, AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; import { isValidAgentProvider, AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
import {
buildVoiceAgentMcpServerConfig,
buildVoiceModeSystemPrompt,
stripVoiceModeSystemPrompt,
} from "./voice-config.js";
import { isVoicePermissionAllowed } from "./voice-permission-policy.js"; import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
import { import {
listDirectoryEntries, listDirectoryEntries,
@@ -130,18 +135,6 @@ const MAX_AGENTS_PER_PROJECT = 5;
* Uses Claude Haiku for speed and cost efficiency. * Uses Claude Haiku for speed and cost efficiency.
*/ */
const AUTO_GEN_MODEL = "haiku"; const AUTO_GEN_MODEL = "haiku";
const VOICE_AGENT_SYSTEM_INSTRUCTION = [
"You are the Paseo voice assistant.",
"The user cannot see your chat messages or tool calls.",
"Always use the speak tool for all user-facing communication.",
"Before calling any non-speak tool, first call speak with a short acknowledgement of what you heard and what you will do next.",
"For long-running work, use speak to provide progress updates before and during execution.",
"Treat the user input as transcribed speech and confirm intent in your first speak response to catch transcription errors early.",
"If the transcription seems incomplete, cut off, or ambiguous, ask a clarifying question via speak before taking action.",
"Use concise plain language suitable for speech output.",
"Never use bash, file-edit, or web tools directly.",
"Only use the paseo MCP tools.",
].join(" ");
function deriveRemoteProjectKey(remoteUrl: string | null): string | null { function deriveRemoteProjectKey(remoteUrl: string | null): string | null {
if (!remoteUrl) { if (!remoteUrl) {
@@ -222,25 +215,6 @@ function escapeXmlText(value: string): string {
.replace(/>/g, "&gt;"); .replace(/>/g, "&gt;");
} }
export function buildVoiceAgentMcpServerConfig(params: {
command: string;
baseArgs: string[];
socketPath: string;
env?: Record<string, string>;
}): {
type: "stdio";
command: string;
args: string[];
env?: Record<string, string>;
} {
return {
type: "stdio",
command: params.command,
args: [...params.baseArgs, "--socket", params.socketPath],
...(params.env ? { env: params.env } : {}),
};
}
type ProcessingPhase = "idle" | "transcribing" | "llm"; type ProcessingPhase = "idle" | "transcribing" | "llm";
type NormalizedGitOptions = { type NormalizedGitOptions = {
@@ -1541,13 +1515,6 @@ export class Session {
return JSON.parse(JSON.stringify(servers)) as Record<string, McpServerConfig>; return JSON.parse(JSON.stringify(servers)) as Record<string, McpServerConfig>;
} }
private buildVoiceModeSystemPrompt(existing?: string): string {
const chunks = [existing?.trim(), VOICE_AGENT_SYSTEM_INSTRUCTION].filter(
(value): value is string => Boolean(value && value.length > 0)
);
return chunks.join("\n\n");
}
private buildVoiceModeMcpServers( private buildVoiceModeMcpServers(
existing: Record<string, McpServerConfig> | undefined, existing: Record<string, McpServerConfig> | undefined,
socketPath: string socketPath: string
@@ -1579,12 +1546,12 @@ export class Session {
this.registerVoiceBridgeForAgent(agentId); this.registerVoiceBridgeForAgent(agentId);
const baseConfig: VoiceModeBaseConfig = { const baseConfig: VoiceModeBaseConfig = {
systemPrompt: existing.config.systemPrompt, systemPrompt: stripVoiceModeSystemPrompt(existing.config.systemPrompt),
mcpServers: this.cloneMcpServers(existing.config.mcpServers), mcpServers: this.cloneMcpServers(existing.config.mcpServers),
}; };
this.voiceModeBaseConfig = baseConfig; this.voiceModeBaseConfig = baseConfig;
const refreshOverrides: Partial<AgentSessionConfig> = { const refreshOverrides: Partial<AgentSessionConfig> = {
systemPrompt: this.buildVoiceModeSystemPrompt(baseConfig.systemPrompt), systemPrompt: buildVoiceModeSystemPrompt(baseConfig.systemPrompt, true),
mcpServers: this.buildVoiceModeMcpServers(baseConfig.mcpServers, socketPath), mcpServers: this.buildVoiceModeMcpServers(baseConfig.mcpServers, socketPath),
}; };
@@ -1625,7 +1592,7 @@ export class Session {
const baseConfig = this.voiceModeBaseConfig; const baseConfig = this.voiceModeBaseConfig;
try { try {
await this.agentManager.refreshAgentFromPersistence(agentId, { await this.agentManager.refreshAgentFromPersistence(agentId, {
systemPrompt: baseConfig.systemPrompt, systemPrompt: buildVoiceModeSystemPrompt(baseConfig.systemPrompt, false),
mcpServers: this.cloneMcpServers(baseConfig.mcpServers), mcpServers: this.cloneMcpServers(baseConfig.mcpServers),
}); });
} catch (error) { } catch (error) {
@@ -1,6 +1,10 @@
import { describe, expect, test } from "vitest"; import { describe, expect, test } from "vitest";
import { buildVoiceAgentMcpServerConfig } from "./session.js"; import {
buildVoiceAgentMcpServerConfig,
buildVoiceModeSystemPrompt,
stripVoiceModeSystemPrompt,
} from "./voice-config.js";
describe("voice MCP stdio config", () => { describe("voice MCP stdio config", () => {
test("builds stdio MCP config for voice agent", () => { test("builds stdio MCP config for voice agent", () => {
@@ -25,3 +29,48 @@ describe("voice MCP stdio config", () => {
}); });
}); });
}); });
describe("voice mode prompt instructions", () => {
test("builds enabled voice instructions and preserves base prompt", () => {
const prompt = buildVoiceModeSystemPrompt("Base system prompt", true);
expect(prompt).toContain("Base system prompt");
expect(prompt).toContain("<paseo_voice_mode>");
expect(prompt).toContain("Paseo voice mode is now on.");
expect(prompt).toContain("Always use the speak tool for all user-facing communication.");
expect(prompt).toContain("</paseo_voice_mode>");
});
test("builds disabled voice instructions and supersedes previous voice block", () => {
const existing = [
"Base system prompt",
"<paseo_voice_mode>",
"legacy voice instruction",
"</paseo_voice_mode>",
].join("\n\n");
const prompt = buildVoiceModeSystemPrompt(existing, false);
expect(prompt).toContain("Base system prompt");
expect(prompt).toContain("Paseo voice mode is now off.");
expect(prompt).toContain("Ignore any earlier Paseo voice mode instructions in this thread.");
expect(prompt.match(/<paseo_voice_mode>/g)?.length ?? 0).toBe(1);
expect(prompt).not.toContain("legacy voice instruction");
});
test("strips voice blocks from persisted prompt", () => {
const existing = [
"Base system prompt",
"<paseo_voice_mode>",
"legacy voice instruction",
"</paseo_voice_mode>",
].join("\n\n");
expect(stripVoiceModeSystemPrompt(existing)).toBe("Base system prompt");
expect(
stripVoiceModeSystemPrompt(
["<paseo_voice_mode>", "legacy voice instruction", "</paseo_voice_mode>"].join("\n\n")
)
).toBeUndefined();
});
});
@@ -0,0 +1,75 @@
const VOICE_PROMPT_BLOCK_START = "<paseo_voice_mode>";
const VOICE_PROMPT_BLOCK_END = "</paseo_voice_mode>";
const VOICE_AGENT_SYSTEM_INSTRUCTION = [
"Paseo voice mode is now on.",
"You are the Paseo voice assistant.",
"The user cannot see your chat messages or tool calls.",
"Always use the speak tool for all user-facing communication.",
"Before calling any non-speak tool, first call speak with a short acknowledgement of what you heard and what you will do next.",
"For long-running work, use speak to provide progress updates before and during execution.",
"Treat the user input as transcribed speech.",
"If the user intent is clear, proceed without extra confirmation.",
"If the transcription seems incomplete, cut off, ambiguous, or may contain a non-obvious mistake or misspelling, ask a clarifying question via speak before taking action.",
"Use concise plain language suitable for speech output.",
].join(" ");
const VOICE_AGENT_DISABLED_INSTRUCTION = [
"Paseo voice mode is now off.",
"Ignore any earlier Paseo voice mode instructions in this thread.",
].join(" ");
function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function makeVoicePromptBlockRegex(): RegExp {
return new RegExp(
`${escapeRegExp(VOICE_PROMPT_BLOCK_START)}[\\s\\S]*?${escapeRegExp(
VOICE_PROMPT_BLOCK_END
)}`,
"g"
);
}
export function stripVoiceModeSystemPrompt(existing?: string): string | undefined {
const trimmed = existing?.trim();
if (!trimmed) {
return undefined;
}
const stripped = trimmed.replace(makeVoicePromptBlockRegex(), "").trim();
return stripped.length > 0 ? stripped : undefined;
}
export function buildVoiceModeSystemPrompt(existing: string | undefined, enabled: boolean): string {
const basePrompt = stripVoiceModeSystemPrompt(existing);
const voiceInstruction = enabled
? VOICE_AGENT_SYSTEM_INSTRUCTION
: VOICE_AGENT_DISABLED_INSTRUCTION;
const voiceBlock = [VOICE_PROMPT_BLOCK_START, voiceInstruction, VOICE_PROMPT_BLOCK_END].join(
"\n"
);
return [basePrompt, voiceBlock]
.filter((entry): entry is string => typeof entry === "string" && entry.length > 0)
.join("\n\n");
}
export function buildVoiceAgentMcpServerConfig(params: {
command: string;
baseArgs: string[];
socketPath: string;
env?: Record<string, string>;
}): {
type: "stdio";
command: string;
args: string[];
env?: Record<string, string>;
} {
return {
type: "stdio",
command: params.command,
args: [...params.baseArgs, "--socket", params.socketPath],
...(params.env ? { env: params.env } : {}),
};
}