feat: improve UI/UX with volume meter and refine input area styling
- Add VolumeMeter component for realtime audio visualization - Refactor input area to show volume meter in realtime mode - Update button styling and layout for cleaner interface - Improve text input integration with better visual hierarchy - Add volume state tracking to Speechmatics audio hook - Add agent working directory validation on creation - Update agent prompt with better orchestration guidelines Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude <noreply@anthropic.com> Co-Authored-By: Happy <yesreply@happy.engineering>
This commit is contained in:
Mohamed Boudra
co-authored by
Claude
Happy
Claude
Happy
parent
de5aba4186
commit
e1e14b1473
+216
-71
@@ -11,11 +11,18 @@ import {
|
||||
Keyboard,
|
||||
} from "react-native";
|
||||
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
|
||||
import ReanimatedAnimated, { useAnimatedStyle } from "react-native-reanimated";
|
||||
import ReanimatedAnimated, {
|
||||
useAnimatedStyle,
|
||||
useSharedValue,
|
||||
withSpring,
|
||||
withRepeat,
|
||||
withSequence,
|
||||
withTiming,
|
||||
} from "react-native-reanimated";
|
||||
import { router } from "expo-router";
|
||||
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
|
||||
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
||||
import { StyleSheet } from "react-native-unistyles";
|
||||
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||
import { theme as defaultTheme } from "../styles/theme";
|
||||
|
||||
// Simple unique ID generator
|
||||
@@ -39,6 +46,7 @@ import { ArtifactDrawer, type Artifact } from "@/components/artifact-drawer";
|
||||
import { ActiveProcesses } from "@/components/active-processes";
|
||||
import { AgentStreamView } from "@/components/agent-stream-view";
|
||||
import { ConversationSelector } from "@/components/conversation-selector";
|
||||
import { VolumeMeter } from "@/components/volume-meter";
|
||||
import {
|
||||
Settings,
|
||||
Mic,
|
||||
@@ -125,6 +133,97 @@ interface AgentUpdate {
|
||||
notification: SessionNotification;
|
||||
}
|
||||
|
||||
interface RealtimeCircleProps {
|
||||
volume: number;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
function RealtimeCircle({ volume, onClose }: RealtimeCircleProps) {
|
||||
const { theme } = useUnistyles();
|
||||
|
||||
// Base size for the circles
|
||||
const BASE_SIZE = 80;
|
||||
const MIN_SCALE = 1;
|
||||
const MAX_SCALE = 1.8;
|
||||
|
||||
// Create animated value for volume-based scaling
|
||||
const volumeScale = useSharedValue(MIN_SCALE);
|
||||
const pulseScale = useSharedValue(1);
|
||||
|
||||
// Update volume scale when volume changes
|
||||
useEffect(() => {
|
||||
// Map volume (0-1) to scale range (MIN_SCALE - MAX_SCALE)
|
||||
const targetScale = MIN_SCALE + (volume * (MAX_SCALE - MIN_SCALE));
|
||||
volumeScale.value = withSpring(targetScale, {
|
||||
damping: 15,
|
||||
stiffness: 150,
|
||||
});
|
||||
}, [volume]);
|
||||
|
||||
// Continuous pulsating animation for outer circle
|
||||
useEffect(() => {
|
||||
pulseScale.value = withRepeat(
|
||||
withSequence(
|
||||
withTiming(1.3, { duration: 1000 }),
|
||||
withTiming(1, { duration: 1000 })
|
||||
),
|
||||
-1,
|
||||
false
|
||||
);
|
||||
}, []);
|
||||
|
||||
// Animated style for inner circle (volume reactive)
|
||||
const innerCircleStyle = useAnimatedStyle(() => ({
|
||||
width: BASE_SIZE,
|
||||
height: BASE_SIZE,
|
||||
borderRadius: BASE_SIZE / 2,
|
||||
transform: [{ scale: volumeScale.value }],
|
||||
}));
|
||||
|
||||
// Animated style for outer circle (continuous pulse)
|
||||
const outerCircleStyle = useAnimatedStyle(() => ({
|
||||
width: BASE_SIZE * 1.5,
|
||||
height: BASE_SIZE * 1.5,
|
||||
borderRadius: (BASE_SIZE * 1.5) / 2,
|
||||
transform: [{ scale: pulseScale.value }],
|
||||
opacity: 0.3,
|
||||
}));
|
||||
|
||||
return (
|
||||
<Pressable
|
||||
onPress={onClose}
|
||||
style={{
|
||||
flex: 1,
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
}}
|
||||
>
|
||||
<View style={{ position: "relative", alignItems: "center", justifyContent: "center" }}>
|
||||
{/* Outer pulsating circle */}
|
||||
<ReanimatedAnimated.View
|
||||
style={[
|
||||
outerCircleStyle,
|
||||
{
|
||||
position: "absolute",
|
||||
backgroundColor: theme.colors.palette.blue[600],
|
||||
},
|
||||
]}
|
||||
/>
|
||||
|
||||
{/* Inner volume-reactive circle */}
|
||||
<ReanimatedAnimated.View
|
||||
style={[
|
||||
innerCircleStyle,
|
||||
{
|
||||
backgroundColor: theme.colors.palette.blue[600],
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</View>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
|
||||
export default function VoiceAssistantScreen() {
|
||||
const { settings, isLoading: settingsLoading } = useSettings();
|
||||
const [conversationId, setConversationId] = useState<string | null>(null);
|
||||
@@ -142,7 +241,7 @@ export default function VoiceAssistantScreen() {
|
||||
const { height: keyboardHeight } = useReanimatedKeyboardAnimation();
|
||||
const bottomInset = insets.bottom;
|
||||
const animatedKeyboardStyle = useAnimatedStyle(() => {
|
||||
'worklet';
|
||||
"worklet";
|
||||
const absoluteHeight = Math.abs(keyboardHeight.value);
|
||||
const padding = Math.max(0, absoluteHeight - bottomInset);
|
||||
return {
|
||||
@@ -1100,64 +1199,95 @@ export default function VoiceAssistantScreen() {
|
||||
{/* Fixed Footer */}
|
||||
<View
|
||||
style={[
|
||||
styles.inputArea,
|
||||
styles.inputAreaWrapper,
|
||||
{ paddingBottom: Math.max(insets.bottom, 8) },
|
||||
]}
|
||||
>
|
||||
{/* Text input */}
|
||||
<TextInput
|
||||
value={userInput}
|
||||
onChangeText={setUserInput}
|
||||
placeholder="Say something..."
|
||||
placeholderTextColor={defaultTheme.colors.mutedForeground}
|
||||
style={styles.textInput}
|
||||
multiline
|
||||
editable={!isRecording && ws.isConnected}
|
||||
/>
|
||||
|
||||
{/* Buttons */}
|
||||
<View style={styles.buttonRow}>
|
||||
{/* Realtime mode button */}
|
||||
{isRealtimeMode ? (
|
||||
// Realtime mode - show volume meter
|
||||
<Pressable
|
||||
onPress={handleRealtimeToggle}
|
||||
disabled={!ws.isConnected}
|
||||
style={[
|
||||
styles.realtimeButton,
|
||||
!ws.isConnected && styles.buttonDisabled,
|
||||
isRealtimeMode && styles.realtimeButtonActive,
|
||||
]}
|
||||
style={[styles.inputArea, { minHeight: 200, justifyContent: "center" }]}
|
||||
>
|
||||
<Animated.View style={{ transform: [{ scale: pulseAnim }] }}>
|
||||
<AudioLines size={20} color="white" />
|
||||
</Animated.View>
|
||||
<VolumeMeter volume={realtimeAudio.volume} />
|
||||
</Pressable>
|
||||
) : (
|
||||
// Normal mode - show text input and buttons
|
||||
<View style={styles.inputArea}>
|
||||
{/* Text input */}
|
||||
<TextInput
|
||||
value={userInput}
|
||||
onChangeText={setUserInput}
|
||||
placeholder="Say something..."
|
||||
placeholderTextColor={defaultTheme.colors.mutedForeground}
|
||||
style={styles.textInput}
|
||||
multiline
|
||||
editable={!isRecording && ws.isConnected}
|
||||
/>
|
||||
|
||||
{/* Main action button */}
|
||||
<Pressable
|
||||
onPress={handleButtonClick}
|
||||
disabled={!ws.isConnected}
|
||||
style={[
|
||||
styles.mainButton,
|
||||
!ws.isConnected && styles.buttonDisabled,
|
||||
isRecording && styles.mainButtonRecording,
|
||||
isInProgress && styles.mainButtonInProgress,
|
||||
userInput.trim() &&
|
||||
!isRecording &&
|
||||
!isInProgress &&
|
||||
styles.mainButtonWithText,
|
||||
]}
|
||||
>
|
||||
{isInProgress ? (
|
||||
<Square size={18} color="white" fill="white" />
|
||||
) : isRecording ? (
|
||||
<Square size={14} color="white" fill="white" />
|
||||
) : userInput.trim() ? (
|
||||
<ArrowUp size={20} color="white" />
|
||||
) : (
|
||||
<Mic size={20} color="white" />
|
||||
)}
|
||||
</Pressable>
|
||||
</View>
|
||||
{/* Buttons */}
|
||||
<View style={styles.buttonRow}>
|
||||
{userInput.trim().length > 0 ? (
|
||||
// Send button when text is entered
|
||||
<Pressable
|
||||
onPress={handleSendMessage}
|
||||
disabled={!ws.isConnected || isInProgress}
|
||||
style={[
|
||||
styles.sendButton,
|
||||
!ws.isConnected && styles.buttonDisabled,
|
||||
]}
|
||||
>
|
||||
<ArrowUp size={20} color="white" />
|
||||
</Pressable>
|
||||
) : (
|
||||
// Record and Realtime buttons when no text
|
||||
<>
|
||||
{/* Main action button */}
|
||||
<Pressable
|
||||
onPress={handleButtonClick}
|
||||
disabled={!ws.isConnected}
|
||||
style={[
|
||||
styles.mainButton,
|
||||
!ws.isConnected && styles.buttonDisabled,
|
||||
isRecording && styles.mainButtonRecording,
|
||||
isInProgress && styles.mainButtonInProgress,
|
||||
]}
|
||||
>
|
||||
{isInProgress ? (
|
||||
<Square size={18} color="white" fill="white" />
|
||||
) : isRecording ? (
|
||||
<Square size={14} color="white" fill="white" />
|
||||
) : (
|
||||
<Mic size={20} color={defaultTheme.colors.foreground} />
|
||||
)}
|
||||
</Pressable>
|
||||
|
||||
{/* Realtime mode button */}
|
||||
<Pressable
|
||||
onPress={handleRealtimeToggle}
|
||||
disabled={!ws.isConnected}
|
||||
style={[
|
||||
styles.realtimeButton,
|
||||
!ws.isConnected && styles.buttonDisabled,
|
||||
isRealtimeMode && styles.realtimeButtonActive,
|
||||
]}
|
||||
>
|
||||
<Animated.View style={{ transform: [{ scale: pulseAnim }] }}>
|
||||
<AudioLines
|
||||
size={20}
|
||||
color={
|
||||
isRealtimeMode
|
||||
? defaultTheme.colors.accentForeground
|
||||
: defaultTheme.colors.background
|
||||
}
|
||||
/>
|
||||
</Animated.View>
|
||||
</Pressable>
|
||||
</>
|
||||
)}
|
||||
</View>
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
</ReanimatedAnimated.View>
|
||||
|
||||
@@ -1221,51 +1351,66 @@ const styles = StyleSheet.create((theme) => ({
|
||||
paddingBottom: theme.spacing[4],
|
||||
flexGrow: 1,
|
||||
},
|
||||
inputArea: {
|
||||
paddingTop: theme.spacing[4],
|
||||
paddingHorizontal: theme.spacing[6],
|
||||
inputAreaWrapper: {
|
||||
borderTopRightRadius: theme.borderRadius["2xl"],
|
||||
borderTopLeftRadius: theme.borderRadius["2xl"],
|
||||
paddingTop: theme.spacing[2],
|
||||
paddingHorizontal: theme.spacing[2],
|
||||
borderTopWidth: theme.borderWidth[1],
|
||||
borderTopColor: theme.colors.border,
|
||||
backgroundColor: theme.colors.background,
|
||||
backgroundColor: theme.colors.muted,
|
||||
},
|
||||
inputArea: {
|
||||
borderRadius: theme.borderRadius["2xl"],
|
||||
paddingHorizontal: theme.spacing[2],
|
||||
paddingVertical: theme.spacing[3],
|
||||
},
|
||||
textInput: {
|
||||
backgroundColor: theme.colors.muted,
|
||||
backgroundColor: "transparent",
|
||||
color: theme.colors.foreground,
|
||||
borderRadius: theme.borderRadius["2xl"],
|
||||
paddingHorizontal: theme.spacing[4],
|
||||
paddingVertical: theme.spacing[3],
|
||||
marginBottom: theme.spacing[4],
|
||||
paddingHorizontal: 0,
|
||||
paddingVertical: 0,
|
||||
fontSize: theme.fontSize.lg,
|
||||
marginBottom: theme.spacing[3],
|
||||
maxHeight: 128,
|
||||
},
|
||||
buttonRow: {
|
||||
flexDirection: "row",
|
||||
alignItems: "center",
|
||||
justifyContent: "flex-end",
|
||||
gap: theme.spacing[3],
|
||||
marginBottom: theme.spacing[2],
|
||||
gap: theme.spacing[2],
|
||||
marginBottom: theme.spacing[1],
|
||||
},
|
||||
realtimeButton: {
|
||||
width: 48,
|
||||
height: 48,
|
||||
width: 40,
|
||||
height: 40,
|
||||
borderRadius: theme.borderRadius.full,
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
backgroundColor: theme.colors.muted,
|
||||
backgroundColor: theme.colors.accentForeground,
|
||||
},
|
||||
realtimeButtonActive: {
|
||||
backgroundColor: theme.colors.palette.blue[600],
|
||||
backgroundColor: theme.colors.foreground,
|
||||
},
|
||||
mainButton: {
|
||||
width: 48,
|
||||
height: 48,
|
||||
width: 40,
|
||||
height: 40,
|
||||
borderRadius: theme.borderRadius.full,
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
backgroundColor: theme.colors.secondary,
|
||||
backgroundColor: "transparent",
|
||||
},
|
||||
mainButtonRecording: {
|
||||
backgroundColor: theme.colors.palette.red[500],
|
||||
},
|
||||
sendButton: {
|
||||
width: 40,
|
||||
height: 40,
|
||||
borderRadius: theme.borderRadius.full,
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
backgroundColor: theme.colors.palette.blue[600],
|
||||
},
|
||||
mainButtonInProgress: {
|
||||
backgroundColor: theme.colors.palette.red[600],
|
||||
},
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import { useEffect } from "react";
|
||||
import { View } from "react-native";
|
||||
import ReanimatedAnimated, {
|
||||
useSharedValue,
|
||||
useAnimatedStyle,
|
||||
withSpring,
|
||||
withRepeat,
|
||||
withSequence,
|
||||
withTiming,
|
||||
Easing,
|
||||
} from "react-native-reanimated";
|
||||
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||
|
||||
interface VolumeMeterProps {
|
||||
volume: number;
|
||||
}
|
||||
|
||||
export function VolumeMeter({ volume }: VolumeMeterProps) {
|
||||
const { theme } = useUnistyles();
|
||||
|
||||
// Base dimensions
|
||||
const LINE_SPACING = 8;
|
||||
const MAX_HEIGHT = 80;
|
||||
const MIN_HEIGHT = 4;
|
||||
|
||||
// Shared values for each line's height
|
||||
const line1Height = useSharedValue(MIN_HEIGHT);
|
||||
const line2Height = useSharedValue(MIN_HEIGHT);
|
||||
const line3Height = useSharedValue(MIN_HEIGHT);
|
||||
|
||||
// Idle pulse animations (when no volume)
|
||||
const line1Pulse = useSharedValue(1);
|
||||
const line2Pulse = useSharedValue(1);
|
||||
const line3Pulse = useSharedValue(1);
|
||||
|
||||
// Start idle animations with different phases
|
||||
useEffect(() => {
|
||||
// Line 1 - fastest pulse
|
||||
line1Pulse.value = withRepeat(
|
||||
withSequence(
|
||||
withTiming(1.2, { duration: 800, easing: Easing.inOut(Easing.ease) }),
|
||||
withTiming(1, { duration: 800, easing: Easing.inOut(Easing.ease) })
|
||||
),
|
||||
-1,
|
||||
false
|
||||
);
|
||||
|
||||
// Line 2 - medium pulse with offset
|
||||
line2Pulse.value = withRepeat(
|
||||
withSequence(
|
||||
withTiming(1, { duration: 400 }),
|
||||
withTiming(1.15, { duration: 1000, easing: Easing.inOut(Easing.ease) }),
|
||||
withTiming(1, { duration: 1000, easing: Easing.inOut(Easing.ease) })
|
||||
),
|
||||
-1,
|
||||
false
|
||||
);
|
||||
|
||||
// Line 3 - slowest pulse with different offset
|
||||
line3Pulse.value = withRepeat(
|
||||
withSequence(
|
||||
withTiming(1, { duration: 600 }),
|
||||
withTiming(1.25, { duration: 1200, easing: Easing.inOut(Easing.ease) }),
|
||||
withTiming(1, { duration: 1200, easing: Easing.inOut(Easing.ease) })
|
||||
),
|
||||
-1,
|
||||
false
|
||||
);
|
||||
}, []);
|
||||
|
||||
// Update heights based on volume with different responsiveness per line
|
||||
useEffect(() => {
|
||||
if (volume > 0.01) {
|
||||
// Active volume - animate heights based on volume
|
||||
// Line 1 - most responsive, follows volume closely
|
||||
const target1 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.2);
|
||||
line1Height.value = withSpring(target1, {
|
||||
damping: 10,
|
||||
stiffness: 200,
|
||||
});
|
||||
|
||||
// Line 2 - medium responsiveness
|
||||
const target2 = MIN_HEIGHT + (MAX_HEIGHT * volume * 0.9);
|
||||
line2Height.value = withSpring(target2, {
|
||||
damping: 12,
|
||||
stiffness: 150,
|
||||
});
|
||||
|
||||
// Line 3 - smoothest, lags behind
|
||||
const target3 = MIN_HEIGHT + (MAX_HEIGHT * volume * 0.7);
|
||||
line3Height.value = withSpring(target3, {
|
||||
damping: 15,
|
||||
stiffness: 100,
|
||||
});
|
||||
} else {
|
||||
// No volume - return to minimum
|
||||
line1Height.value = withSpring(MIN_HEIGHT, {
|
||||
damping: 20,
|
||||
stiffness: 150,
|
||||
});
|
||||
line2Height.value = withSpring(MIN_HEIGHT, {
|
||||
damping: 20,
|
||||
stiffness: 150,
|
||||
});
|
||||
line3Height.value = withSpring(MIN_HEIGHT, {
|
||||
damping: 20,
|
||||
stiffness: 150,
|
||||
});
|
||||
}
|
||||
}, [volume]);
|
||||
|
||||
// Animated styles for each line
|
||||
const line1Style = useAnimatedStyle(() => ({
|
||||
height: line1Height.value * (volume > 0.01 ? 1 : line1Pulse.value),
|
||||
opacity: 0.8 + (volume * 0.2),
|
||||
}));
|
||||
|
||||
const line2Style = useAnimatedStyle(() => ({
|
||||
height: line2Height.value * (volume > 0.01 ? 1 : line2Pulse.value),
|
||||
opacity: 0.7 + (volume * 0.3),
|
||||
}));
|
||||
|
||||
const line3Style = useAnimatedStyle(() => ({
|
||||
height: line3Height.value * (volume > 0.01 ? 1 : line3Pulse.value),
|
||||
opacity: 0.6 + (volume * 0.4),
|
||||
}));
|
||||
|
||||
return (
|
||||
<View style={styles.container}>
|
||||
<ReanimatedAnimated.View
|
||||
style={[
|
||||
styles.line,
|
||||
{
|
||||
width: 4,
|
||||
backgroundColor: theme.colors.palette.blue[400],
|
||||
},
|
||||
line1Style,
|
||||
]}
|
||||
/>
|
||||
<View style={{ width: LINE_SPACING }} />
|
||||
<ReanimatedAnimated.View
|
||||
style={[
|
||||
styles.line,
|
||||
{
|
||||
width: 6,
|
||||
backgroundColor: theme.colors.palette.blue[500],
|
||||
},
|
||||
line2Style,
|
||||
]}
|
||||
/>
|
||||
<View style={{ width: LINE_SPACING }} />
|
||||
<ReanimatedAnimated.View
|
||||
style={[
|
||||
styles.line,
|
||||
{
|
||||
width: 4,
|
||||
backgroundColor: theme.colors.palette.blue[400],
|
||||
},
|
||||
line3Style,
|
||||
]}
|
||||
/>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create((theme) => ({
|
||||
container: {
|
||||
flexDirection: "row",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
height: 100,
|
||||
},
|
||||
line: {
|
||||
borderRadius: theme.borderRadius.full,
|
||||
},
|
||||
}));
|
||||
@@ -25,6 +25,7 @@ export interface SpeechmaticsAudio {
|
||||
isActive: boolean;
|
||||
isSpeaking: boolean;
|
||||
isDetecting: boolean;
|
||||
volume: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -129,6 +130,7 @@ export function useSpeechmaticsAudio(
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
const [isDetecting, setIsDetecting] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
const [volume, setVolume] = useState(0);
|
||||
|
||||
const audioBufferRef = useRef<Uint8Array[]>([]);
|
||||
const silenceStartRef = useRef<number | null>(null);
|
||||
@@ -188,17 +190,18 @@ export function useSpeechmaticsAudio(
|
||||
(event) => {
|
||||
if (!isActive) return;
|
||||
|
||||
const volume: number = event.data;
|
||||
const speechDetected = volume > VOLUME_THRESHOLD;
|
||||
const volumeLevel: number = event.data;
|
||||
setVolume(volumeLevel);
|
||||
const speechDetected = volumeLevel > VOLUME_THRESHOLD;
|
||||
|
||||
// console.log('[SpeechmaticsAudio] Volume:', volume.toFixed(6));
|
||||
// console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD);
|
||||
|
||||
if (speechDetected && !isSpeakingRef.current && !speechConfirmedRef.current) {
|
||||
// Initial speech detection - start tracking
|
||||
if (speechDetectionStartRef.current === null) {
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech started (volume:",
|
||||
volume.toFixed(4),
|
||||
volumeLevel.toFixed(4),
|
||||
")"
|
||||
);
|
||||
speechDetectionStartRef.current = Date.now();
|
||||
@@ -329,6 +332,7 @@ export function useSpeechmaticsAudio(
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
setVolume(0);
|
||||
|
||||
console.log("[SpeechmaticsAudio] Audio capture stopped");
|
||||
}
|
||||
@@ -348,5 +352,6 @@ export function useSpeechmaticsAudio(
|
||||
isActive,
|
||||
isSpeaking,
|
||||
isDetecting,
|
||||
volume,
|
||||
};
|
||||
}
|
||||
|
||||
+237
-66
@@ -337,102 +337,194 @@ present_artifact({
|
||||
|
||||
## 5. Claude Code Integration
|
||||
|
||||
### Your Role: Orchestrator
|
||||
|
||||
**You are a high-level orchestrator, not a code executor.**
|
||||
|
||||
Your job is to:
|
||||
- Understand user intent and delegate work to coding agents
|
||||
- Maintain context of active agents and conversations
|
||||
- Handle quick one-off commands yourself
|
||||
- Coordinate between agents, terminals, and git operations
|
||||
|
||||
**Delegation Philosophy:**
|
||||
- **Complex coding work** → Create agent with initial prompt and mode
|
||||
- **Quick info/operations** → Execute directly yourself
|
||||
- **Active agent context** → Send prompts to existing agent
|
||||
|
||||
**Example workflow:**
|
||||
|
||||
```
|
||||
User: "Add authentication to the API"
|
||||
You: "Starting agent to add authentication."
|
||||
[create_coding_agent with initialPrompt and bypassPermissions mode]
|
||||
[Agent works autonomously, you monitor]
|
||||
|
||||
User: "What's it doing?"
|
||||
You: [get_agent_activity to check progress]
|
||||
You: "Adding JWT middleware and login endpoint."
|
||||
|
||||
[Agent completes]
|
||||
You: "Authentication added with JWT tokens and login endpoint."
|
||||
|
||||
User: "Add tests for that"
|
||||
You: "Asking agent to add tests."
|
||||
[send_agent_prompt to same agent - maintains context]
|
||||
|
||||
User: "Commit and push"
|
||||
You: "Asking agent to commit and push."
|
||||
[send_agent_prompt - agent handles git operations]
|
||||
```
|
||||
|
||||
**Key: You orchestrate. Agents execute. Context matters.**
|
||||
|
||||
### What is Claude Code?
|
||||
|
||||
Command-line AI coding agent. Launch it like any other command:
|
||||
Claude Code is an AI coding agent that can handle complex coding tasks. Delegate work to it by creating agents with clear instructions.
|
||||
|
||||
### Creating Agents
|
||||
|
||||
**Best Practice: Always create with initialPrompt and initialMode**
|
||||
|
||||
This allows the agent to start working immediately. You just wait and check the results.
|
||||
|
||||
```javascript
|
||||
execute_command("claude", "~/dev/voice-dev")
|
||||
// ✅ RECOMMENDED: Agent starts working immediately
|
||||
create_coding_agent({
|
||||
cwd: "~/dev/voice-dev",
|
||||
initialPrompt: "add dark mode toggle to settings page",
|
||||
initialMode: "bypassPermissions" // Auto-approve all actions
|
||||
})
|
||||
// Agent starts working right away, you monitor progress
|
||||
|
||||
// ✅ For planning/review: Use plan mode
|
||||
create_coding_agent({
|
||||
cwd: "~/dev/project",
|
||||
initialPrompt: "refactor authentication module",
|
||||
initialMode: "plan" // Shows plan before executing
|
||||
})
|
||||
|
||||
// ⚠️ Less common: Create without initial task
|
||||
create_coding_agent({
|
||||
cwd: "~/dev/voice-dev"
|
||||
})
|
||||
// Agent waits idle, requires send_agent_prompt to start work
|
||||
```
|
||||
|
||||
### Vim Mode Input
|
||||
**Available modes:**
|
||||
- `"default"` - Asks permission for each action (slow for voice)
|
||||
- `"acceptEdits"` - Auto-approves file edits, asks for commands
|
||||
- `"plan"` - Shows plan before executing
|
||||
- `"bypassPermissions"` - Auto-approves everything (fastest, recommended for most tasks)
|
||||
|
||||
Claude Code uses Vim keybindings:
|
||||
### Working with Agents
|
||||
|
||||
- `-- INSERT --` visible = insert mode (can type freely)
|
||||
- No `-- INSERT --` visible = normal mode (press `i` to enter insert)
|
||||
|
||||
### Permission Modes
|
||||
|
||||
Cycle through 4 modes with **Shift+Tab** (BTab):
|
||||
|
||||
1. **Default** (no indicator) - Asks permission for everything
|
||||
2. **⏵⏵ accept edits on** - Auto-accepts file edits only
|
||||
3. **⏸ plan mode on** - Shows plan before executing
|
||||
4. **⏵⏵ bypass permissions on** - Auto-executes ALL actions
|
||||
|
||||
**Efficient mode switching:**
|
||||
**Send prompts to agents:**
|
||||
|
||||
```javascript
|
||||
// To plan mode from default (2 presses)
|
||||
send_keys_to_command(commandId, "BTab", repeat=2, return_output={lines: 50})
|
||||
// Send task (non-blocking by default)
|
||||
send_agent_prompt({
|
||||
agentId: "abc123",
|
||||
prompt: "explain how authentication works"
|
||||
})
|
||||
// Returns immediately, agent processes in background
|
||||
|
||||
// To bypass from default (3 presses)
|
||||
send_keys_to_command(commandId, "BTab", repeat=3, return_output={lines: 50})
|
||||
// Send task and wait for completion
|
||||
send_agent_prompt({
|
||||
agentId: "abc123",
|
||||
prompt: "fix the bug in auth.ts",
|
||||
maxWait: 60000 // Wait up to 60 seconds
|
||||
})
|
||||
|
||||
// Change mode and send prompt
|
||||
send_agent_prompt({
|
||||
agentId: "abc123",
|
||||
prompt: "implement user registration",
|
||||
sessionMode: "bypassPermissions" // Auto-approve all actions
|
||||
})
|
||||
```
|
||||
|
||||
### Basic Claude Code Workflow
|
||||
|
||||
**Starting:**
|
||||
**Check agent status:**
|
||||
|
||||
```javascript
|
||||
// Basic launch
|
||||
execute_command("claude", "~/dev/project")
|
||||
// Get current status
|
||||
get_agent_status({ agentId: "abc123" })
|
||||
// Returns: { status: "processing", info: {...} }
|
||||
|
||||
// With initial prompt
|
||||
execute_command('claude "add dark mode toggle"', "~/dev/project")
|
||||
// Get agent activity (curated, human-readable)
|
||||
get_agent_activity({
|
||||
agentId: "abc123",
|
||||
format: "curated" // Clean summary of what agent did
|
||||
})
|
||||
|
||||
// In plan mode
|
||||
execute_command("claude --permission-mode plan", "~/dev/project")
|
||||
// List all agents
|
||||
list_agents()
|
||||
// Returns: { agents: [{id, status, createdAt, ...}, ...] }
|
||||
```
|
||||
|
||||
**Asking a question:**
|
||||
**Control agents:**
|
||||
|
||||
```javascript
|
||||
// 1. Check for "-- INSERT --" in output from list_commands or capture_command
|
||||
// 2. If not in insert mode, enter it:
|
||||
send_keys_to_command(commandId, "i", return_output={lines: 20})
|
||||
// 3. Type question:
|
||||
send_text_to_command(
|
||||
commandId,
|
||||
"explain how authentication works",
|
||||
pressEnter=true,
|
||||
return_output={lines: 50, maxWait: 5000}
|
||||
)
|
||||
// Change session mode
|
||||
set_agent_mode({
|
||||
agentId: "abc123",
|
||||
modeId: "plan" // Switch to plan mode
|
||||
})
|
||||
// Available modes: default, acceptEdits, plan, bypassPermissions
|
||||
|
||||
// Cancel current task (agent stays alive)
|
||||
cancel_agent({ agentId: "abc123" })
|
||||
|
||||
// Kill agent completely
|
||||
kill_agent({ agentId: "abc123" })
|
||||
```
|
||||
|
||||
**Closing:**
|
||||
### Agent Creation Patterns
|
||||
|
||||
#### Pattern 1: Quick Task
|
||||
|
||||
```javascript
|
||||
// Graceful exit
|
||||
send_text_to_command(commandId, "/exit", pressEnter=true)
|
||||
|
||||
// Force quit
|
||||
send_keys_to_command(commandId, "C-c", repeat=2)
|
||||
// Create agent with task in one step
|
||||
create_coding_agent({
|
||||
cwd: "~/dev/faro/main",
|
||||
initialPrompt: "refactor the authentication module"
|
||||
})
|
||||
```
|
||||
|
||||
### Launching Claude Code - Patterns
|
||||
|
||||
#### Pattern 1: Basic Launch
|
||||
#### Pattern 2: Create and Monitor
|
||||
|
||||
```javascript
|
||||
execute_command("claude", "~/dev/faro/main")
|
||||
// 1. Create agent
|
||||
const result = create_coding_agent({ cwd: "~/dev/project" })
|
||||
const agentId = result.agentId
|
||||
|
||||
// 2. Send task
|
||||
send_agent_prompt({
|
||||
agentId: agentId,
|
||||
prompt: "add unit tests for the API"
|
||||
})
|
||||
|
||||
// 3. Check progress later
|
||||
get_agent_activity({ agentId: agentId })
|
||||
```
|
||||
|
||||
#### Pattern 2: Launch with Worktree
|
||||
#### Pattern 3: Worktree Workflow
|
||||
|
||||
```javascript
|
||||
// 1. Create worktree
|
||||
const result = execute_command(
|
||||
const worktreeResult = execute_command(
|
||||
"create-worktree fix-auth",
|
||||
"~/dev/voice-dev",
|
||||
maxWait=5000
|
||||
)
|
||||
|
||||
// 2. Parse WORKTREE_PATH from result.output
|
||||
// 2. Parse WORKTREE_PATH from worktreeResult.output
|
||||
|
||||
// 3. Launch Claude in worktree
|
||||
execute_command("claude", worktreePath)
|
||||
// 3. Create agent in worktree
|
||||
create_coding_agent({
|
||||
cwd: worktreePath,
|
||||
initialPrompt: "fix authentication bug"
|
||||
})
|
||||
```
|
||||
|
||||
## 6. Git & GitHub
|
||||
@@ -479,23 +571,102 @@ All projects in `~/dev`:
|
||||
|
||||
### Context-Aware Execution
|
||||
|
||||
**When to use Claude Code:**
|
||||
- Coding tasks (refactoring, adding features, fixing bugs)
|
||||
- Already working with Claude Code on a task
|
||||
- Context clues: "add feature", "refactor this", "fix bug"
|
||||
**Maintain conversation context:**
|
||||
|
||||
You must track:
|
||||
- Which agents are active and what they're working on
|
||||
- What directory/project context was established in the conversation
|
||||
- Whether user is continuing work with an existing agent
|
||||
|
||||
**Context-based decision making:**
|
||||
|
||||
**Scenario 1: Fresh session, no context**
|
||||
```
|
||||
User: "Run git status"
|
||||
You: "Which project?" (or present options)
|
||||
```
|
||||
|
||||
**Scenario 2: Agent is active in a directory**
|
||||
```
|
||||
User: "Run git status"
|
||||
You: [execute_command in the agent's working directory - context is clear]
|
||||
```
|
||||
|
||||
**Scenario 3: Just created agent in ~/dev/faro**
|
||||
```
|
||||
User: "Run git status"
|
||||
You: [execute_command in ~/dev/faro - we established context]
|
||||
```
|
||||
|
||||
**When to create new agents:**
|
||||
- Complex coding tasks requiring codebase understanding
|
||||
- Multi-step work (refactoring, adding features, fixing bugs)
|
||||
- Context clues: "add feature", "refactor this", "fix bug", "implement"
|
||||
- **Best practice: Create with initialPrompt and initialMode so agent starts immediately**
|
||||
|
||||
**When to use existing agent:**
|
||||
- Agent is already working in the relevant directory
|
||||
- User is continuing a conversation with the agent
|
||||
- Task relates to agent's current work
|
||||
- Examples: "commit that", "add tests for this", "explain what you changed"
|
||||
|
||||
**When to execute directly:**
|
||||
- Quick info gathering (git status, ls, grep)
|
||||
- Simple operations (git commands, gh commands)
|
||||
- Claude Code not involved
|
||||
- Context clues: "check status", "run tests", "create PR"
|
||||
- Quick one-off commands (git status, ls, grep)
|
||||
- Simple git/gh operations when no agent is involved
|
||||
- Reading files or showing information
|
||||
- Context clues: "check status", "show me", "what's in"
|
||||
- **Exception: If agent is active and request relates to its work, delegate to agent**
|
||||
|
||||
**Delegation examples:**
|
||||
|
||||
```javascript
|
||||
// ✅ Delegate coding work with clear instructions
|
||||
User: "Add dark mode to the settings page"
|
||||
You: "Starting agent to add dark mode."
|
||||
create_coding_agent({
|
||||
cwd: "~/dev/voice-dev",
|
||||
initialPrompt: "Add dark mode toggle to the settings page",
|
||||
initialMode: "bypassPermissions" // Start working immediately
|
||||
})
|
||||
|
||||
// ✅ Continue with active agent
|
||||
User: "Now add tests for that"
|
||||
You: "Asking agent to add tests."
|
||||
send_agent_prompt({
|
||||
agentId: activeAgentId,
|
||||
prompt: "Add unit tests for the dark mode feature"
|
||||
})
|
||||
|
||||
// ✅ Delegate git operations to active agent
|
||||
User: "Commit and push that"
|
||||
You: "Asking agent to commit and push."
|
||||
send_agent_prompt({
|
||||
agentId: activeAgentId,
|
||||
prompt: "Create a git commit for these changes and push to remote"
|
||||
})
|
||||
|
||||
// ✅ Execute simple command yourself
|
||||
User: "What's the current branch?"
|
||||
You: [execute_command("git branch --show-current", agentWorkingDir)]
|
||||
You: "You're on main."
|
||||
|
||||
// ❌ Don't execute git operations if agent should handle it
|
||||
User: "Commit and push that" [agent just finished work]
|
||||
You: [execute_command("git add .")] // WRONG - delegate to agent instead
|
||||
```
|
||||
|
||||
**Key principle: If there's an active agent working on something and the user asks to do related work (commit, test, modify, etc.), send the request to that agent rather than executing commands yourself.**
|
||||
|
||||
### Remember
|
||||
|
||||
- **You are an orchestrator** - delegate complex work to coding agents
|
||||
- **Track context** - remember active agents, working directories, conversation flow
|
||||
- **Agent-first for coding** - create agents with initialPrompt + initialMode for immediate work
|
||||
- **Delegate to active agents** - if agent is working, send related tasks to that agent
|
||||
- **Execute simple tasks yourself** - quick git commands, file reads, status checks
|
||||
- **ALWAYS call the actual tool** - never just describe what you would do
|
||||
- **1-3 sentences max** - voice users process info differently
|
||||
- **Progressive disclosure** - answer what's asked, wait for follow-ups
|
||||
- **Use context** - fix STT errors silently, infer ambiguous references
|
||||
- **Use context** - fix STT errors silently, infer ambiguous references from conversation
|
||||
- **Always report results** - voice users can't see command output
|
||||
- **Commands stay available** - finished commands can be inspected until killed
|
||||
- **Default to action** - when in doubt, make best guess and execute
|
||||
|
||||
@@ -166,6 +166,16 @@ describe("AgentManager", () => {
|
||||
// ==========================================================================
|
||||
|
||||
describe("Agent Creation and Directory Control", () => {
|
||||
it("should reject creation when working directory does not exist", async () => {
|
||||
const nonExistentDir = join(tmpDir, "does-not-exist");
|
||||
|
||||
await expect(
|
||||
manager.createAgent({
|
||||
cwd: nonExistentDir,
|
||||
})
|
||||
).rejects.toThrow(/Working directory does not exist or is not accessible/);
|
||||
});
|
||||
|
||||
it(
|
||||
"should create agent and run in specified directory",
|
||||
async () => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { spawn, ChildProcess } from "child_process";
|
||||
import { Writable, Readable } from "stream";
|
||||
import { access, constants } from "fs/promises";
|
||||
import {
|
||||
ClientSideConnection,
|
||||
ndJsonStream,
|
||||
@@ -102,6 +103,15 @@ export class AgentManager {
|
||||
const agentId = uuidv4();
|
||||
const cwd = options.cwd;
|
||||
|
||||
// Validate that the working directory exists
|
||||
try {
|
||||
await access(cwd, constants.R_OK | constants.X_OK);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Working directory does not exist or is not accessible: ${cwd}`
|
||||
);
|
||||
}
|
||||
|
||||
// Spawn the ACP process
|
||||
const agentProcess = spawn("npx", ["@zed-industries/claude-code-acp"], {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
|
||||
Reference in New Issue
Block a user