feat: implement realtime audio with VAD for Android

- Install react-native-vad and expo-audio-studio for voice detection - Create use-realtime-audio hook for continuous recording + VAD segmentation - Add Android FOREGROUND_SERVICE permissions for background mic access - Update UI components with Lucide icons and markdown rendering - Add npm overrides to resolve peer dependency conflicts with Expo SDK 54 - Integrate realtime mode toggle with pulse animation for speech detection Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude <noreply@anthropic.com> Co-Authored-By: Happy <yesreply@happy.engineering>
2025-10-21 21:33:14 +02:00
parent cc3dbb88cb
commit 0f1450381a
9 changed files with 1013 additions and 177 deletions
@@ -1,11 +1,14 @@
 import { useAudioRecorder as useExpoAudioRecorder, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
 import { Paths, File, Directory, FileInfo } from 'expo-file-system';
-import { useState } from 'react';
+import { useState, useEffect } from 'react';

 export interface AudioCaptureConfig {
  sampleRate?: number;
  numberOfChannels?: number;
  bitRate?: number;
+  onAudioLevel?: (level: number) => void;
+  onSpeechSegment?: (audioBlob: Blob) => void;
+  enableContinuousRecording?: boolean;
 }

 /**
@@ -133,6 +136,7 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
    numberOfChannels: config?.numberOfChannels || 1,
    bitRate: config?.bitRate || 128000,
    extension: '.m4a',
+    isMeteringEnabled: !!config?.onAudioLevel, // Enable metering if callback provided
    android: {
      extension: '.m4a',
      outputFormat: 'mpeg4',
@@ -156,6 +160,24 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {

  const audioRecorder = useExpoAudioRecorder(recordingOptions);

+  // Monitor audio levels if metering is enabled
+  useEffect(() => {
+    if (!config?.onAudioLevel || !isRecording) return;
+
+    const interval = setInterval(() => {
+      const metering = audioRecorder.state?.metering;
+      if (metering !== undefined && metering !== null) {
+        // Normalize metering value (typically ranges from -160 to 0 dB)
+        // Convert to 0-1 range where 0 is silence and 1 is loud
+        // We'll use -40 dB as the threshold for "loud"
+        const normalized = Math.max(0, Math.min(1, (metering + 40) / 40));
+        config.onAudioLevel(normalized);
+      }
+    }, 100); // Check every 100ms
+
+    return () => clearInterval(interval);
+  }, [isRecording, config, audioRecorder.state?.metering]);
+
  async function start(): Promise<void> {
    if (isRecording) {
      throw new Error('Already recording');
@@ -0,0 +1,269 @@
+import { useState, useEffect, useRef } from 'react';
+import { Platform } from 'react-native';
+import { createVADRNBridgeInstance } from 'react-native-vad';
+import { AudioRecording } from '@siteed/expo-audio-studio';
+import { check, request, PERMISSIONS, RESULTS } from 'react-native-permissions';
+
+export interface RealtimeAudioConfig {
+  onSpeechStart?: () => void;
+  onSpeechEnd?: () => void;
+  onAudioSegment?: (audioData: string, format: string) => void;
+  onError?: (error: Error) => void;
+  speechThreshold?: number;
+  silenceDuration?: number;
+}
+
+export interface RealtimeAudio {
+  start: () => Promise<void>;
+  stop: () => Promise<void>;
+  isActive: boolean;
+  isSpeechDetected: boolean;
+}
+
+/**
+ * Hook for realtime audio with VAD-based segmentation
+ * Uses react-native-vad for speech detection and expo-audio-studio for recording
+ */
+export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
+  const [isActive, setIsActive] = useState(false);
+  const [isSpeechDetected, setIsSpeechDetected] = useState(false);
+
+  const vadInstanceRef = useRef<any>(null);
+  const recordingRef = useRef<AudioRecording | null>(null);
+  const audioBufferRef = useRef<string[]>([]);
+  const vadPollingIntervalRef = useRef<NodeJS.Timeout | null>(null);
+  const isSpeakingRef = useRef(false);
+  const silenceStartRef = useRef<number | null>(null);
+
+  const SPEECH_THRESHOLD = config.speechThreshold ?? 0.5;
+  const SILENCE_DURATION_MS = config.silenceDuration ?? 1000; // 1 second of silence to end segment
+
+  async function requestMicrophonePermission(): Promise<boolean> {
+    try {
+      const permission = Platform.OS === 'ios'
+        ? PERMISSIONS.IOS.MICROPHONE
+        : PERMISSIONS.ANDROID.RECORD_AUDIO;
+
+      const status = await check(permission);
+      if (status === RESULTS.GRANTED) {
+        return true;
+      }
+
+      const result = await request(permission);
+      return result === RESULTS.GRANTED;
+    } catch (error) {
+      console.error('[RealtimeAudio] Permission error:', error);
+      return false;
+    }
+  }
+
+  async function initializeVAD(): Promise<void> {
+    try {
+      console.log('[RealtimeAudio] Initializing VAD...');
+      const instance = await createVADRNBridgeInstance('realtime-vad', false);
+      if (!instance) {
+        throw new Error('Failed to create VAD instance');
+      }
+
+      // Create VAD instance with sensitivity
+      instance.createInstance(0.1, 10);
+
+      // Set license (using the example license from the demo)
+      await instance.setVADDetectionLicense(
+        "MTczOTU3MDQwMDAwMA==-+2/cH2HBQz3/SsDidS6qvIgc8KxGH5cbvSVM/6qmk3Q="
+      );
+
+      // Start VAD detection
+      await instance.startVADDetection();
+
+      vadInstanceRef.current = instance;
+      console.log('[RealtimeAudio] VAD initialized successfully');
+    } catch (error) {
+      console.error('[RealtimeAudio] VAD initialization error:', error);
+      throw error;
+    }
+  }
+
+  async function startAudioRecording(): Promise<void> {
+    try {
+      console.log('[RealtimeAudio] Starting audio recording...');
+
+      const recording = new AudioRecording({
+        sampleRate: 16000,
+        channels: 1,
+        encoding: 'pcm_16bit',
+        interval: 500, // Get audio chunks every 500ms
+      });
+
+      // Handle audio stream chunks
+      recording.onAudioStream = (event: any) => {
+        // Only buffer chunks while speech is detected
+        if (isSpeakingRef.current && event.data) {
+          audioBufferRef.current.push(event.data);
+          console.log('[RealtimeAudio] Buffered chunk, total chunks:', audioBufferRef.current.length);
+        }
+      };
+
+      await recording.start();
+      recordingRef.current = recording;
+      console.log('[RealtimeAudio] Audio recording started');
+    } catch (error) {
+      console.error('[RealtimeAudio] Recording start error:', error);
+      throw error;
+    }
+  }
+
+  function startVADPolling(): void {
+    console.log('[RealtimeAudio] Starting VAD polling...');
+
+    vadPollingIntervalRef.current = setInterval(async () => {
+      if (!vadInstanceRef.current) return;
+
+      try {
+        const voiceProps = await vadInstanceRef.current.getVoiceProps();
+        const probability = voiceProps.voiceProbability;
+        const speechDetected = probability > SPEECH_THRESHOLD;
+
+        // Update UI state
+        setIsSpeechDetected(speechDetected);
+
+        // Handle speech start
+        if (speechDetected && !isSpeakingRef.current) {
+          console.log('[RealtimeAudio] Speech START detected (probability:', probability, ')');
+          isSpeakingRef.current = true;
+          silenceStartRef.current = null;
+          audioBufferRef.current = []; // Start new buffer
+          config.onSpeechStart?.();
+        }
+        // Handle potential speech end (silence detected)
+        else if (!speechDetected && isSpeakingRef.current) {
+          if (silenceStartRef.current === null) {
+            // First silence detection - start timer
+            silenceStartRef.current = Date.now();
+          } else {
+            // Check if silence duration threshold reached
+            const silenceDuration = Date.now() - silenceStartRef.current;
+            if (silenceDuration >= SILENCE_DURATION_MS) {
+              console.log('[RealtimeAudio] Speech END detected (silence for', silenceDuration, 'ms)');
+              isSpeakingRef.current = false;
+              silenceStartRef.current = null;
+              config.onSpeechEnd?.();
+
+              // Send buffered audio segment
+              if (audioBufferRef.current.length > 0) {
+                const combinedAudio = combineAudioChunks(audioBufferRef.current);
+                config.onAudioSegment?.(combinedAudio, 'audio/wav');
+                audioBufferRef.current = [];
+              }
+            }
+          }
+        }
+        // Reset silence timer if speech detected again
+        else if (speechDetected && isSpeakingRef.current && silenceStartRef.current !== null) {
+          silenceStartRef.current = null;
+        }
+      } catch (error) {
+        console.error('[RealtimeAudio] VAD polling error:', error);
+      }
+    }, 200); // Poll every 200ms
+  }
+
+  function combineAudioChunks(chunks: string[]): string {
+    // Combine base64 PCM chunks into a single audio segment
+    // For now, just concatenate - you may need proper WAV header
+    console.log('[RealtimeAudio] Combining', chunks.length, 'audio chunks');
+    return chunks.join('');
+  }
+
+  async function start(): Promise<void> {
+    if (isActive) {
+      console.log('[RealtimeAudio] Already active');
+      return;
+    }
+
+    try {
+      console.log('[RealtimeAudio] Starting realtime audio...');
+
+      // Request permissions
+      const hasPermission = await requestMicrophonePermission();
+      if (!hasPermission) {
+        throw new Error('Microphone permission denied');
+      }
+
+      // Initialize VAD
+      await initializeVAD();
+
+      // Start audio recording
+      await startAudioRecording();
+
+      // Start VAD polling
+      startVADPolling();
+
+      setIsActive(true);
+      console.log('[RealtimeAudio] Realtime audio started successfully');
+    } catch (error) {
+      console.error('[RealtimeAudio] Start error:', error);
+      const err = error instanceof Error ? error : new Error(String(error));
+      config.onError?.(err);
+
+      // Cleanup on error
+      await stop();
+      throw error;
+    }
+  }
+
+  async function stop(): Promise<void> {
+    console.log('[RealtimeAudio] Stopping realtime audio...');
+
+    // Stop VAD polling
+    if (vadPollingIntervalRef.current) {
+      clearInterval(vadPollingIntervalRef.current);
+      vadPollingIntervalRef.current = null;
+    }
+
+    // Stop recording
+    if (recordingRef.current) {
+      try {
+        await recordingRef.current.stop();
+        recordingRef.current = null;
+      } catch (error) {
+        console.error('[RealtimeAudio] Error stopping recording:', error);
+      }
+    }
+
+    // Stop VAD
+    if (vadInstanceRef.current) {
+      try {
+        await vadInstanceRef.current.stopVADDetection();
+        vadInstanceRef.current = null;
+      } catch (error) {
+        console.error('[RealtimeAudio] Error stopping VAD:', error);
+      }
+    }
+
+    // Reset state
+    audioBufferRef.current = [];
+    isSpeakingRef.current = false;
+    silenceStartRef.current = null;
+    setIsActive(false);
+    setIsSpeechDetected(false);
+
+    console.log('[RealtimeAudio] Realtime audio stopped');
+  }
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      if (isActive) {
+        stop();
+      }
+    };
+  }, []);
+
+  return {
+    start,
+    stop,
+    isActive,
+    isSpeechDetected,
+  };
+}