jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

```

Browse Source 
feat: add realtime mode drift protection and improve voice UX
```
This commit is contained in:
Mohamed Boudra
2025-10-22 19:06:59 +02:00
parent 015d0f0e27
commit 8b9561b379
6 changed files with 308 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/app/index.tsx
+148 -56
View File
@@ -53,6 +53,7 @@ import {
ArrowUp,
Square,
AudioLines,
MicOff,
} from "lucide-react-native";
import type {
ActivityLogPayload,
@@ -231,8 +232,14 @@ export default function VoiceAssistantScreen() {
// Realtime mode state (defined early so we can use it in audioRecorder)
const [isRealtimeMode, setIsRealtimeMode] = useState(false);
const isRealtimeModeRef = useRef(isRealtimeMode);
const pulseAnim = useRef(new Animated.Value(1)).current;
// Keep ref in sync with state
useEffect(() => {
isRealtimeModeRef.current = isRealtimeMode;
}, [isRealtimeMode]);
const insets = useSafeAreaInsets();
const audioRecorder = useAudioRecorder();
const audioPlayer = useAudioPlayer();
@@ -293,7 +300,7 @@ export default function VoiceAssistantScreen() {
]);
},
volumeThreshold: 0.3,
silenceDuration: 1000,
silenceDuration: 2000,
speechConfirmationDuration: 300,
detectionGracePeriod: 200,
});
@@ -452,6 +459,11 @@ export default function VoiceAssistantScreen() {
if (message.type !== "activity_log") return;
const data = message.payload;
// Filter out transcription activity logs
if (data.type === "system" && data.content.includes("Transcribing")) {
return;
}
// Handle tool calls
if (data.type === "tool_call" && data.metadata) {
const {
@@ -593,6 +605,42 @@ export default function VoiceAssistantScreen() {
if (message.type !== "audio_output") return;
const data = message.payload;
const currentIsRealtimeMode = isRealtimeModeRef.current;
// Drift protection: Don't play audio generated in different mode
if (data.isRealtimeMode !== currentIsRealtimeMode) {
console.log(
`[App] Skipping audio playback due to mode drift (generated in ${data.isRealtimeMode ? "realtime" : "normal"} mode, currently in ${currentIsRealtimeMode ? "realtime" : "normal"} mode)`
);
// Still send confirmation to prevent server from waiting
const confirmMessage: WSInboundMessage = {
type: "session",
message: {
type: "audio_played",
id: data.id,
},
};
ws.send(confirmMessage);
return;
}
// Additional check: Don't play if NOT in realtime mode (shouldn't happen with server-side fix, but defense in depth)
if (!currentIsRealtimeMode) {
console.log("[App] Skipping audio playback - not in realtime mode");
// Still send confirmation
const confirmMessage: WSInboundMessage = {
type: "session",
message: {
type: "audio_played",
id: data.id,
},
};
ws.send(confirmMessage);
return;
}
try {
setIsPlayingAudio(true);
@@ -669,18 +717,7 @@ export default function VoiceAssistantScreen() {
// Conversation loaded handler
const unsubConversationLoaded = ws.on("conversation_loaded", (message) => {
if (message.type !== "conversation_loaded") return;
const { conversationId, messageCount } = message.payload;
setMessages((prev) => [
...prev,
{
type: "activity",
id: generateMessageId(),
timestamp: Date.now(),
activityType: "success",
message: `Loaded conversation with ${messageCount} messages`,
},
]);
// Don't show message in UI
});
// Artifact handler
@@ -734,32 +771,6 @@ export default function VoiceAssistantScreen() {
};
}, [ws, audioPlayer]);
// Connection status handler
useEffect(() => {
if (ws.isConnected) {
setMessages((prev) => [
...prev,
{
type: "activity",
id: generateMessageId(),
timestamp: Date.now(),
activityType: "success",
message: "WebSocket connected",
},
]);
} else if (messages.length > 0) {
setMessages((prev) => [
...prev,
{
type: "activity",
id: generateMessageId(),
timestamp: Date.now(),
activityType: "error",
message: "WebSocket disconnected",
},
]);
}
}, [ws.isConnected]);
// Voice button handler
async function handleVoicePress() {
@@ -778,16 +789,6 @@ export default function VoiceAssistantScreen() {
);
setIsProcessingAudio(true);
setMessages((prev) => [
...prev,
{
type: "activity",
id: generateMessageId(),
timestamp: Date.now(),
activityType: "info",
message: "Sending audio to server...",
},
]);
// Convert to base64
const arrayBuffer = await audioBlob.arrayBuffer();
@@ -974,6 +975,16 @@ export default function VoiceAssistantScreen() {
setIsRealtimeMode(true);
console.log("[App] Realtime mode enabled");
// Notify server of mode change
const modeMessage: WSInboundMessage = {
type: "session",
message: {
type: "set_realtime_mode",
enabled: true,
},
};
ws.send(modeMessage);
setMessages((prev) => [
...prev,
{
@@ -1004,6 +1015,16 @@ export default function VoiceAssistantScreen() {
setIsRealtimeMode(false);
console.log("[App] Realtime mode disabled");
// Notify server of mode change
const modeMessage: WSInboundMessage = {
type: "session",
message: {
type: "set_realtime_mode",
enabled: false,
},
};
ws.send(modeMessage);
setMessages((prev) => [
...prev,
{
@@ -1204,13 +1225,49 @@ export default function VoiceAssistantScreen() {
]}
>
{isRealtimeMode ? (
// Realtime mode - show volume meter
<Pressable
onPress={handleRealtimeToggle}
style={[styles.inputArea, { minHeight: 200, justifyContent: "center" }]}
>
<VolumeMeter volume={realtimeAudio.volume} />
</Pressable>
// Realtime mode - show volume meter and mute button
<View style={[styles.inputArea, { minHeight: 200 }]}>
<View style={{ flex: 1, justifyContent: "center", alignItems: "center" }}>
<VolumeMeter
volume={realtimeAudio.volume}
isMuted={realtimeAudio.isMuted}
isDetecting={realtimeAudio.isDetecting}
isSpeaking={realtimeAudio.isSpeaking}
/>
{/* Debug timer */}
{(realtimeAudio.isDetecting || realtimeAudio.isSpeaking) && (
<Text style={styles.debugTimer}>
{(realtimeAudio.segmentDuration / 1000).toFixed(1)}s
</Text>
)}
</View>
<View style={styles.realtimeModeButtons}>
{/* Mute button */}
<Pressable
onPress={() => realtimeAudio.toggleMute()}
style={[
styles.realtimeMuteButton,
realtimeAudio.isMuted && styles.realtimeMuteButtonActive,
]}
>
<MicOff
size={20}
color={
realtimeAudio.isMuted
? defaultTheme.colors.background
: defaultTheme.colors.foreground
}
/>
</Pressable>
{/* Close button */}
<Pressable
onPress={handleRealtimeToggle}
style={styles.realtimeCloseButton}
>
<Square size={18} color="white" fill="white" />
</Pressable>
</View>
</View>
) : (
// Normal mode - show text input and buttons
<View style={styles.inputArea}>
@@ -1420,4 +1477,39 @@ const styles = StyleSheet.create((theme) => ({
buttonDisabled: {
opacity: theme.opacity[50],
},
realtimeModeButtons: {
flexDirection: "row",
alignItems: "center",
justifyContent: "center",
gap: theme.spacing[3],
paddingTop: theme.spacing[4],
},
realtimeMuteButton: {
width: 48,
height: 48,
borderRadius: theme.borderRadius.full,
alignItems: "center",
justifyContent: "center",
backgroundColor: theme.colors.muted,
borderWidth: theme.borderWidth[2],
borderColor: theme.colors.border,
},
realtimeMuteButtonActive: {
backgroundColor: theme.colors.palette.red[500],
borderColor: theme.colors.palette.red[600],
},
realtimeCloseButton: {
width: 48,
height: 48,
borderRadius: theme.borderRadius.full,
alignItems: "center",
justifyContent: "center",
backgroundColor: theme.colors.palette.red[600],
},
debugTimer: {
marginTop: theme.spacing[2],
color: theme.colors.mutedForeground,
fontSize: theme.fontSize.sm,
fontFamily: "monospace",
},
}));
packages/app/src/components/volume-meter.tsx
+60 -23
View File
@@ -13,15 +13,18 @@ import { StyleSheet, useUnistyles } from "react-native-unistyles";
interface VolumeMeterProps {
volume: number;
isMuted?: boolean;
isDetecting?: boolean;
isSpeaking?: boolean;
}
export function VolumeMeter({ volume }: VolumeMeterProps) {
export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSpeaking = false }: VolumeMeterProps) {
const { theme } = useUnistyles();
// Base dimensions
const LINE_SPACING = 8;
const MAX_HEIGHT = 80;
const MIN_HEIGHT = 4;
const MAX_HEIGHT = 50;
const MIN_HEIGHT = 20;
// Shared values for each line's height
const line1Height = useSharedValue(MIN_HEIGHT);
@@ -35,6 +38,14 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
// Start idle animations with different phases
useEffect(() => {
if (isMuted) {
// When muted, set pulse to 1 (no animation)
line1Pulse.value = 1;
line2Pulse.value = 1;
line3Pulse.value = 1;
return;
}
// Line 1 - fastest pulse
line1Pulse.value = withRepeat(
withSequence(
@@ -66,10 +77,18 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
-1,
false
);
}, []);
}, [isMuted]);
// Update heights based on volume with different responsiveness per line
useEffect(() => {
if (isMuted) {
// When muted, keep lines at minimum height without animation
line1Height.value = MIN_HEIGHT;
line2Height.value = MIN_HEIGHT;
line3Height.value = MIN_HEIGHT;
return;
}
if (volume > 0.01) {
// Active volume - animate heights based on volume
// Line 1 - most responsive, follows volume closely
@@ -107,23 +126,41 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
stiffness: 150,
});
}
}, [volume]);
}, [volume, isMuted]);
// Animated styles for each line
const line1Style = useAnimatedStyle(() => ({
height: line1Height.value * (volume > 0.01 ? 1 : line1Pulse.value),
opacity: 0.8 + (volume * 0.2),
}));
const line1Style = useAnimatedStyle(() => {
const isActive = isDetecting || isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted ? 0 : volume * 0.3;
return {
height: line1Height.value * (isMuted || volume > 0.01 ? 1 : line1Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
const line2Style = useAnimatedStyle(() => ({
height: line2Height.value * (volume > 0.01 ? 1 : line2Pulse.value),
opacity: 0.7 + (volume * 0.3),
}));
const line2Style = useAnimatedStyle(() => {
const isActive = isDetecting || isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted ? 0 : volume * 0.3;
return {
height: line2Height.value * (isMuted || volume > 0.01 ? 1 : line2Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
const line3Style = useAnimatedStyle(() => ({
height: line3Height.value * (volume > 0.01 ? 1 : line3Pulse.value),
opacity: 0.6 + (volume * 0.4),
}));
const line3Style = useAnimatedStyle(() => {
const isActive = isDetecting || isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted ? 0 : volume * 0.3;
return {
height: line3Height.value * (isMuted || volume > 0.01 ? 1 : line3Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
const lineColor = "#FFFFFF";
const lineWidth = 8;
return (
<View style={styles.container}>
@@ -131,8 +168,8 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
style={[
styles.line,
{
width: 4,
backgroundColor: theme.colors.palette.blue[400],
width: lineWidth,
backgroundColor: lineColor,
},
line1Style,
]}
@@ -142,8 +179,8 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
style={[
styles.line,
{
width: 6,
backgroundColor: theme.colors.palette.blue[500],
width: lineWidth,
backgroundColor: lineColor,
},
line2Style,
]}
@@ -153,8 +190,8 @@ export function VolumeMeter({ volume }: VolumeMeterProps) {
style={[
styles.line,
{
width: 4,
backgroundColor: theme.colors.palette.blue[400],
width: lineWidth,
backgroundColor: lineColor,
},
line3Style,
]}
packages/app/src/hooks/use-speechmatics-audio.ts
+52 -3
View File
@@ -22,10 +22,13 @@ export interface SpeechmaticsAudioConfig {
export interface SpeechmaticsAudio {
start: () => Promise<void>;
stop: () => Promise<void>;
toggleMute: () => void;
isActive: boolean;
isSpeaking: boolean;
isDetecting: boolean;
isMuted: boolean;
volume: number;
segmentDuration: number;
}
/**
@@ -131,6 +134,8 @@ export function useSpeechmaticsAudio(
const [isDetecting, setIsDetecting] = useState(false);
const [audioInitialized, setAudioInitialized] = useState(false);
const [volume, setVolume] = useState(0);
const [isMuted, setIsMuted] = useState(false);
const [segmentDuration, setSegmentDuration] = useState(0);
const audioBufferRef = useRef<Uint8Array[]>([]);
const silenceStartRef = useRef<number | null>(null);
@@ -164,12 +169,28 @@ export function useSpeechmaticsAudio(
};
}, []);
// Update segment duration timer
useEffect(() => {
if (!isDetecting && !isSpeaking) {
setSegmentDuration(0);
return;
}
const startTime = speechDetectionStartRef.current || Date.now();
const interval = setInterval(() => {
const elapsed = Date.now() - startTime;
setSegmentDuration(elapsed);
}, 100);
return () => clearInterval(interval);
}, [isDetecting, isSpeaking]);
// Listen to microphone data
useExpoTwoWayAudioEventListener(
"onMicrophoneData",
useCallback<MicrophoneDataCallback>(
(event) => {
if (!isActive) return;
if (!isActive || isMuted) return;
const pcmData: Uint8Array = event.data;
@@ -179,7 +200,7 @@ export function useSpeechmaticsAudio(
audioBufferRef.current.push(pcmData);
}
},
[isActive]
[isActive, isMuted]
)
);
@@ -192,6 +213,9 @@ export function useSpeechmaticsAudio(
const volumeLevel: number = event.data;
setVolume(volumeLevel);
if (isMuted) return;
const speechDetected = volumeLevel > VOLUME_THRESHOLD;
// console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD);
@@ -285,7 +309,7 @@ export function useSpeechmaticsAudio(
}
}
},
[isActive, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config]
[isActive, isMuted, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config]
)
);
@@ -333,10 +357,32 @@ export function useSpeechmaticsAudio(
setIsSpeaking(false);
setIsDetecting(false);
setVolume(0);
setIsMuted(false);
console.log("[SpeechmaticsAudio] Audio capture stopped");
}
function toggleMute(): void {
setIsMuted((prev) => {
const newMuted = !prev;
console.log("[SpeechmaticsAudio] Mute toggled:", newMuted);
if (newMuted) {
// Clear any ongoing speech detection/speaking state
audioBufferRef.current = [];
isSpeakingRef.current = false;
speechConfirmedRef.current = false;
speechDetectionStartRef.current = null;
detectionSilenceStartRef.current = null;
silenceStartRef.current = null;
setIsSpeaking(false);
setIsDetecting(false);
}
return newMuted;
});
}
// Cleanup on unmount
useEffect(() => {
return () => {
@@ -349,9 +395,12 @@ export function useSpeechmaticsAudio(
return {
start,
stop,
toggleMute,
isActive,
isSpeaking,
isDetecting,
isMuted,
volume,
segmentDuration,
};
}