jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement realtime audio with VAD for Android

Browse Source 
- Install react-native-vad and expo-audio-studio for voice detection
- Create use-realtime-audio hook for continuous recording + VAD segmentation
- Add Android FOREGROUND_SERVICE permissions for background mic access
- Update UI components with Lucide icons and markdown rendering
- Add npm overrides to resolve peer dependency conflicts with Expo SDK 54
- Integrate realtime mode toggle with pulse animation for speech detection

Generated with [Claude Code](https://claude.ai/code)
via [Happy](https://happy.engineering)

Co-Authored-By: Claude <noreply@anthropic.com>
Co-Authored-By: Happy <yesreply@happy.engineering>
This commit is contained in:
Mohamed Boudra
2025-10-21 21:33:14 +02:00
co-authored by Claude Happy
parent cc3dbb88cb
commit 0f1450381a
9 changed files with 1013 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/voice-mobile/components/artifact-drawer.tsx
+4 -3
View File
@@ -1,4 +1,5 @@
import { View, Text, ScrollView, Pressable, Modal } from 'react-native';
import { SafeAreaView } from 'react-native-safe-area-context';
import { useEffect } from 'react';
export interface Artifact {
@@ -42,9 +43,9 @@ export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) {
presentationStyle="pageSheet"
onRequestClose={onClose}
>
<View className="flex-1 bg-black">
<SafeAreaView edges={['top', 'bottom']} className="flex-1 bg-black">
{/* Header */}
<View className="pt-16 pb-4 px-4 border-b border-zinc-800">
<View className="pb-4 px-4 border-b border-zinc-800">
<View className="flex-row items-center justify-between">
<View className="flex-1 mr-4">
<Text className="text-white text-xl font-bold" numberOfLines={2}>
@@ -120,7 +121,7 @@ export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) {
</View>
</View>
</ScrollView>
</View>
</SafeAreaView>
</Modal>
);
}
packages/voice-mobile/components/conversation-selector.tsx
+73 -53
View File
@@ -1,6 +1,7 @@
import { useState, useEffect } from 'react';
import { Modal, View, Text, Pressable, ScrollView, Alert } from 'react-native';
import { MaterialIcons } from '@expo/vector-icons';
import { Modal, View, Text, Pressable, ScrollView, Alert, KeyboardAvoidingView, Platform } from 'react-native';
import { SafeAreaView } from 'react-native-safe-area-context';
import { MessageSquare, X, Plus, Trash2 } from 'lucide-react-native';
import AsyncStorage from '@react-native-async-storage/async-storage';
import type { UseWebSocketReturn } from '../hooks/use-websocket';
@@ -30,7 +31,7 @@ export function ConversationSelector({
// Listen for conversation list responses
useEffect(() => {
const unsubscribe = websocket.on('list_conversations_response', (message) => {
console.log('[ConversationSelector] Received conversations:', message.payload.conversations.length);
if (message.type !== 'list_conversations_response') return;
setConversations(message.payload.conversations);
setIsLoading(false);
});
@@ -41,6 +42,7 @@ export function ConversationSelector({
// Listen for delete conversation responses
useEffect(() => {
const unsubscribe = websocket.on('delete_conversation_response', (message) => {
if (message.type !== 'delete_conversation_response') return;
console.log('[ConversationSelector] Delete response:', message.payload);
if (message.payload.success) {
// Refresh conversations list
@@ -177,7 +179,7 @@ export function ConversationSelector({
onPress={() => setIsOpen(true)}
className="bg-zinc-800 px-4 py-2 rounded-lg"
>
<MaterialIcons name="chat-bubble-outline" size={20} color="white" />
<MessageSquare size={20} color="white" />
</Pressable>
<Modal
@@ -186,17 +188,20 @@ export function ConversationSelector({
transparent={true}
onRequestClose={() => setIsOpen(false)}
>
<View className="flex-1 bg-black/50">
<View style={{ flex: 1, backgroundColor: 'rgba(0,0,0,0.5)' }}>
<Pressable
className="flex-1"
style={{ flex: 1 }}
onPress={() => setIsOpen(false)}
/>
<View className="bg-zinc-900 rounded-t-3xl max-h-[80%]">
{/* Header */}
<View className="flex-row items-center justify-between p-6 border-b border-zinc-800">
<Text className="text-white text-lg font-semibold">Conversations</Text>
<View style={{ backgroundColor: '#18181b', borderTopLeftRadius: 24, borderTopRightRadius: 24, height: '80%' }}>
<View style={{ flex: 1 }}>
{/* Header */}
<View className="flex-row items-center justify-between p-6 border-b border-zinc-800">
<Text className="text-white text-lg font-semibold">
Conversations {conversations.length > 0 && `(${conversations.length})`}
</Text>
<Pressable onPress={() => setIsOpen(false)}>
<MaterialIcons name="close" size={24} color="white" />
<X size={24} color="white" />
</Pressable>
</View>
@@ -206,62 +211,77 @@ export function ConversationSelector({
onPress={handleNewConversation}
className="flex-row items-center justify-center gap-2 bg-zinc-800 py-3 rounded-lg"
>
<MaterialIcons name="add" size={20} color="white" />
<Plus size={20} color="white" />
<Text className="text-white font-semibold">New Conversation</Text>
</Pressable>
</View>
{/* Conversations List */}
<ScrollView className="flex-1">
{isLoading ? (
<ScrollView
style={{ flex: 1 }}
contentContainerStyle={{ paddingBottom: 20 }}
keyboardShouldPersistTaps="handled"
keyboardDismissMode="on-drag"
>
{isLoading && (
<View className="p-8 items-center">
<Text className="text-zinc-400">Loading...</Text>
</View>
) : conversations.length > 0 ? (
<>
{conversations.map((conversation) => (
<Pressable
key={conversation.id}
onPress={() => handleSelectConversation(conversation.id)}
className={`flex-row items-center justify-between p-4 border-b border-zinc-800 ${
conversation.id === currentConversationId ? 'bg-zinc-800' : ''
}`}
>
<View className="flex-1">
<Text className="text-white font-semibold">
{conversation.messageCount} messages
</Text>
<Text className="text-zinc-400 text-sm mt-1">
{formatDate(conversation.lastUpdated)}
</Text>
</View>
<Pressable
onPress={() => handleDeleteConversation(conversation.id)}
className="p-2"
hitSlop={{ top: 10, bottom: 10, left: 10, right: 10 }}
>
<MaterialIcons name="delete-outline" size={20} color="#ef4444" />
</Pressable>
</Pressable>
))}
)}
{/* Clear All Button */}
<View className="p-4">
<Pressable
onPress={handleClearAll}
className="flex-row items-center justify-center gap-2 bg-red-900/20 py-3 rounded-lg border border-red-900"
>
<MaterialIcons name="delete-outline" size={18} color="#ef4444" />
<Text className="text-red-500 font-semibold">Clear All</Text>
</Pressable>
</View>
</>
) : (
{!isLoading && conversations.length === 0 && (
<View className="p-8 items-center">
<Text className="text-zinc-400">No saved conversations</Text>
</View>
)}
{!isLoading && conversations.map((conversation) => (
<View
key={conversation.id}
style={{
flexDirection: 'row',
alignItems: 'center',
justifyContent: 'space-between',
padding: 16,
borderBottomWidth: 1,
borderBottomColor: '#27272a',
backgroundColor: conversation.id === currentConversationId ? '#27272a' : 'transparent'
}}
>
<Pressable
style={{ flex: 1 }}
onPress={() => handleSelectConversation(conversation.id)}
>
<Text style={{ color: 'white', fontWeight: '600', fontSize: 16 }}>
{conversation.messageCount} messages
</Text>
<Text style={{ color: '#a1a1aa', fontSize: 14, marginTop: 4 }}>
{formatDate(conversation.lastUpdated)}
</Text>
</Pressable>
<Pressable
onPress={() => handleDeleteConversation(conversation.id)}
style={{ padding: 8 }}
hitSlop={{ top: 10, bottom: 10, left: 10, right: 10 }}
>
<Trash2 size={20} color="#ef4444" />
</Pressable>
</View>
))}
{!isLoading && conversations.length > 0 && (
<View className="p-4">
<Pressable
onPress={handleClearAll}
className="flex-row items-center justify-center gap-2 bg-red-900/20 py-3 rounded-lg border border-red-900"
>
<Trash2 size={18} color="#ef4444" />
<Text className="text-red-500 font-semibold">Clear All</Text>
</Pressable>
</View>
)}
</ScrollView>
</View>
</View>
</View>
</Modal>
packages/voice-mobile/components/message.tsx
+94 -49
View File
@@ -1,5 +1,7 @@
import { View, Text, Pressable, Animated } from 'react-native';
import { useState, useEffect, useRef } from 'react';
import Markdown from 'react-native-markdown-display';
import { Circle, Info, CheckCircle, XCircle, FileText, ChevronRight, ChevronDown, RefreshCw } from 'lucide-react-native';
interface UserMessageProps {
message: string;
@@ -7,13 +9,10 @@ interface UserMessageProps {
}
export function UserMessage({ message, timestamp }: UserMessageProps) {
const time = new Date(timestamp).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
return (
<View className="flex-row justify-end mb-3 px-4">
<View className="bg-blue-600 rounded-2xl rounded-tr-sm px-4 py-3 max-w-[80%]">
<Text className="text-white text-base leading-5">{message}</Text>
<Text className="text-blue-200 text-xs mt-1">{time}</Text>
<Text className="text-white text-base leading-6">{message}</Text>
</View>
</View>
);
@@ -26,7 +25,6 @@ interface AssistantMessageProps {
}
export function AssistantMessage({ message, timestamp, isStreaming = false }: AssistantMessageProps) {
const time = new Date(timestamp).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
const fadeAnim = useRef(new Animated.Value(0.3)).current;
useEffect(() => {
@@ -51,19 +49,69 @@ export function AssistantMessage({ message, timestamp, isStreaming = false }: As
}
}, [isStreaming, fadeAnim]);
const markdownStyles = {
body: {
color: '#f0fdfa',
fontSize: 16,
lineHeight: 24,
},
paragraph: {
marginTop: 0,
marginBottom: 8,
},
strong: {
fontWeight: '700',
},
em: {
fontStyle: 'italic',
},
code_inline: {
backgroundColor: '#134e4a',
color: '#ccfbf1',
paddingHorizontal: 4,
paddingVertical: 2,
borderRadius: 3,
fontFamily: 'monospace',
},
code_block: {
backgroundColor: '#134e4a',
color: '#ccfbf1',
padding: 12,
borderRadius: 6,
fontFamily: 'monospace',
fontSize: 14,
},
fence: {
backgroundColor: '#134e4a',
color: '#ccfbf1',
padding: 12,
borderRadius: 6,
fontFamily: 'monospace',
fontSize: 14,
},
link: {
color: '#5eead4',
textDecorationLine: 'underline',
},
bullet_list: {
marginBottom: 8,
},
ordered_list: {
marginBottom: 8,
},
list_item: {
marginBottom: 4,
},
};
return (
<View className="flex-row justify-start mb-3 px-4">
<View className="bg-teal-800 rounded-2xl rounded-tl-sm px-4 py-3 max-w-[80%]">
<Text className="text-teal-50 text-base leading-5">{message}</Text>
<View className="flex-row items-center justify-between mt-1">
<Text className="text-teal-200 text-xs">{time}</Text>
{isStreaming && (
<Animated.View style={{ opacity: fadeAnim }}>
<Text className="text-teal-200 text-xs ml-2 font-bold">...</Text>
</Animated.View>
)}
</View>
</View>
<View className="mb-3 px-4 py-3">
<Markdown style={markdownStyles}>{message}</Markdown>
{isStreaming && (
<Animated.View style={{ opacity: fadeAnim }} className="mt-1">
<Text className="text-teal-200 text-xs font-bold">...</Text>
</Animated.View>
)}
</View>
);
}
@@ -92,14 +140,15 @@ export function ActivityLog({
const [isExpanded, setIsExpanded] = useState(false);
const typeConfig = {
system: { bg: 'bg-zinc-800/50', text: 'text-zinc-400', icon: '●' },
info: { bg: 'bg-blue-900/30', text: 'text-blue-400', icon: 'i' },
success: { bg: 'bg-green-900/30', text: 'text-green-400', icon: '✓' },
error: { bg: 'bg-red-900/30', text: 'text-red-400', icon: '✗' },
artifact: { bg: 'bg-blue-900/40', text: 'text-blue-300', icon: '📋' },
system: { bg: 'bg-zinc-800/50', color: '#a1a1aa', Icon: Circle },
info: { bg: 'bg-blue-900/30', color: '#60a5fa', Icon: Info },
success: { bg: 'bg-green-900/30', color: '#4ade80', Icon: CheckCircle },
error: { bg: 'bg-red-900/30', color: '#f87171', Icon: XCircle },
artifact: { bg: 'bg-blue-900/40', color: '#93c5fd', Icon: FileText },
};
const config = typeConfig[type];
const IconComponent = config.Icon;
const handlePress = () => {
if (type === 'artifact' && artifactId && onArtifactClick) {
@@ -123,19 +172,19 @@ export function ActivityLog({
>
<View className="px-3 py-2.5">
<View className="flex-row items-start gap-2">
<Text className={`${config.text} text-sm leading-5`}>
{config.icon}
</Text>
<IconComponent size={16} color={config.color} />
<View className="flex-1">
<Text className={`${config.text} text-sm leading-5`}>
<Text style={{ color: config.color }} className="text-sm leading-5">
{displayMessage}
</Text>
{metadata && (
<View className="flex-row items-center mt-1">
<Text className="text-zinc-500 text-xs">Details</Text>
<Text className="text-zinc-500 text-xs ml-1">
{isExpanded ? '▼' : '▶'}
</Text>
<Text className="text-zinc-500 text-xs mr-1">Details</Text>
{isExpanded ? (
<ChevronDown size={12} color="#71717a" />
) : (
<ChevronRight size={12} color="#71717a" />
)}
</View>
)}
</View>
@@ -188,22 +237,19 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp
executing: {
border: 'border-amber-500',
bg: 'bg-amber-500/20',
text: 'text-amber-300',
icon: '⟳',
color: '#fcd34d',
label: 'executing',
},
completed: {
border: 'border-green-500',
bg: 'bg-green-500/20',
text: 'text-green-300',
icon: '✓',
color: '#86efac',
label: 'completed',
},
failed: {
border: 'border-red-500',
bg: 'bg-red-500/20',
text: 'text-red-300',
icon: '✗',
color: '#fca5a5',
label: 'failed',
},
};
@@ -217,26 +263,25 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp
>
<View className="p-3">
<View className="flex-row items-center">
<Text className="text-zinc-400 text-sm mr-2">
{isExpanded ? '▼' : '▶'}
</Text>
{isExpanded ? (
<ChevronDown size={16} color="#9ca3af" className="mr-2" />
) : (
<ChevronRight size={16} color="#9ca3af" className="mr-2" />
)}
<Text className="text-slate-200 font-mono font-medium text-sm flex-1">
{toolName}
</Text>
<View className={`flex-row items-center gap-1.5 px-2 py-1 rounded ${config.bg}`}>
{status === 'executing' ? (
<Animated.Text
style={{ transform: [{ rotate: spin }] }}
className={`${config.text} text-xs`}
>
{config.icon}
</Animated.Text>
<Animated.View style={{ transform: [{ rotate: spin }] }}>
<RefreshCw size={14} color={config.color} />
</Animated.View>
) : status === 'completed' ? (
<CheckCircle size={14} color={config.color} />
) : (
<Text className={`${config.text} text-xs`}>
{config.icon}
</Text>
<XCircle size={14} color={config.color} />
)}
<Text className={`${config.text} text-xs font-medium uppercase`}>
<Text style={{ color: config.color }} className="text-xs font-medium uppercase">
{config.label}
</Text>
</View>