feat: migrate from NativeWind to react-native-unistyles and add Speechmatics audio

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

packages/app/hooks/use-realtime-audio.ts

@@ -10,6 +10,7 @@ export interface RealtimeAudioConfig {
   onError?: (error: Error) => void;
   volumeThreshold?: number; // RMS threshold for speech detection (0-1)
   silenceDuration?: number; // ms of silence before ending segment
+  speechStartDuration?: number; // ms of sustained volume before starting segment
 }
 
 export interface RealtimeAudio {
@@ -19,6 +20,115 @@ export interface RealtimeAudio {
   isSpeaking: boolean;
 }
 
+/**
+ * Convert raw PCM data to WAV format by adding WAV headers
+ * @param pcmBase64 Base64-encoded PCM data
+ * @param sampleRate Sample rate (default: 16000)
+ * @param channels Number of channels (default: 1 for mono)
+ * @param bitsPerSample Bits per sample (default: 16)
+ * @returns Base64-encoded WAV data
+ */
+function pcmToWav(
+  pcmBase64: string,
+  sampleRate: number = 16000,
+  channels: number = 1,
+  bitsPerSample: number = 16
+): string {
+  // Decode base64 PCM data
+  const pcmBinary = atob(pcmBase64);
+  const pcmLength = pcmBinary.length;
+
+  // Calculate WAV parameters
+  const byteRate = sampleRate * channels * (bitsPerSample / 8);
+  const blockAlign = channels * (bitsPerSample / 8);
+  const dataSize = pcmLength;
+  const fileSize = 44 + dataSize; // WAV header is 44 bytes
+
+  // Create WAV buffer
+  const wavBuffer = new Uint8Array(fileSize);
+  const view = new DataView(wavBuffer.buffer);
+
+  // Write WAV header
+  let offset = 0;
+
+  // RIFF chunk descriptor
+  writeString(view, offset, 'RIFF'); offset += 4;
+  view.setUint32(offset, fileSize - 8, true); offset += 4; // File size - 8
+  writeString(view, offset, 'WAVE'); offset += 4;
+
+  // fmt sub-chunk
+  writeString(view, offset, 'fmt '); offset += 4;
+  view.setUint32(offset, 16, true); offset += 4; // Subchunk size (16 for PCM)
+  view.setUint16(offset, 1, true); offset += 2; // Audio format (1 = PCM)
+  view.setUint16(offset, channels, true); offset += 2;
+  view.setUint32(offset, sampleRate, true); offset += 4;
+  view.setUint32(offset, byteRate, true); offset += 4;
+  view.setUint16(offset, blockAlign, true); offset += 2;
+  view.setUint16(offset, bitsPerSample, true); offset += 2;
+
+  // data sub-chunk
+  writeString(view, offset, 'data'); offset += 4;
+  view.setUint32(offset, dataSize, true); offset += 4;
+
+  // Copy PCM data
+  for (let i = 0; i < pcmLength; i++) {
+    wavBuffer[offset + i] = pcmBinary.charCodeAt(i);
+  }
+
+  // Convert to base64
+  let binary = '';
+  for (let i = 0; i < wavBuffer.length; i++) {
+    binary += String.fromCharCode(wavBuffer[i]);
+  }
+  return btoa(binary);
+}
+
+/**
+ * Helper to write string to DataView
+ */
+function writeString(view: DataView, offset: number, str: string): void {
+  for (let i = 0; i < str.length; i++) {
+    view.setUint8(offset + i, str.charCodeAt(i));
+  }
+}
+
+/**
+ * Decode base64 string to Uint8Array
+ */
+function decodeBase64ToUint8Array(base64: string): Uint8Array {
+  const binary = atob(base64);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+  return bytes;
+}
+
+/**
+ * Encode Uint8Array to base64 string
+ */
+function uint8ArrayToBase64(bytes: Uint8Array): string {
+  let binary = '';
+  for (let i = 0; i < bytes.length; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return btoa(binary);
+}
+
+/**
+ * Concatenate multiple Uint8Arrays into one
+ */
+function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
+  const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const arr of arrays) {
+    result.set(arr, offset);
+    offset += arr.length;
+  }
+  return result;
+}
+
 /**
  * Hook for realtime audio streaming with volume-based VAD
  * Uses expo-audio-studio for recording with echo cancellation and noise suppression
@@ -28,13 +138,17 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
   const [isSpeaking, setIsSpeaking] = useState(false);
 
   const audioRecorder = useAudioRecorder();
-  const audioBufferRef = useRef<string[]>([]);
+  const audioBufferRef = useRef<Uint8Array[]>([]);
   const silenceStartRef = useRef<number | null>(null);
   const isSpeakingRef = useRef(false);
+  const speechStartRef = useRef<number | null>(null); // Track when volume first exceeded threshold
+  const speechConfirmedRef = useRef(false); // Track if speech has been confirmed
   const lastVolumeCheckRef = useRef<number>(Date.now());
+  const lastVolumeLogRef = useRef<number>(Date.now());
 
-  const VOLUME_THRESHOLD = config.volumeThreshold ?? 0.02;
+  const VOLUME_THRESHOLD = config.volumeThreshold ?? 0.2;
   const SILENCE_DURATION_MS = config.silenceDuration ?? 1000;
+  const SPEECH_START_DURATION_MS = config.speechStartDuration ?? 1000;
   const VOLUME_CHECK_INTERVAL = 200; // Check volume every 200ms
 
   async function requestMicrophonePermission(): Promise<boolean> {
@@ -84,7 +198,6 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
       }
 
       const rms = Math.sqrt(sum / sampleCount);
-      console.log('[RealtimeAudio] Volume:', rms.toFixed(6));
       return rms;
     } catch (error) {
       console.error('[RealtimeAudio] Volume calculation error:', error);
@@ -126,38 +239,84 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
             const volume = calculateVolume(audioData);
             const speechDetected = volume > VOLUME_THRESHOLD;
 
-            if (speechDetected && !isSpeakingRef.current) {
-              // Speech START
-              console.log('[RealtimeAudio] Speech started (volume:', volume.toFixed(4), ')');
-              isSpeakingRef.current = true;
+            // Decode base64 chunk to binary for buffering
+            const decodedChunk = decodeBase64ToUint8Array(audioData);
+
+            // Check if we're in detection phase (volume detected but not yet confirmed)
+            const inDetectionPhase = speechStartRef.current !== null && !isSpeakingRef.current;
+
+            if (inDetectionPhase) {
+              // We're in detection phase - check timer regardless of current volume
+              const sustainedDuration = Date.now() - speechStartRef.current!;
+
+              if (sustainedDuration >= SPEECH_START_DURATION_MS) {
+                // Timer elapsed - confirm speech
+                console.log('[RealtimeAudio] ✅ Speech confirmed after', sustainedDuration, 'ms (volume:', volume.toFixed(4), 'threshold:', VOLUME_THRESHOLD, ')');
+                isSpeakingRef.current = true;
+                speechConfirmedRef.current = true;
+                setIsSpeaking(true);
+                config.onSpeechStart?.();
+                audioBufferRef.current.push(decodedChunk);
+                silenceStartRef.current = null;
+              } else if (speechDetected) {
+                // Still in detection phase, volume still above threshold, keep buffering
+                audioBufferRef.current.push(decodedChunk);
+
+                // Log progress every 500ms during detection
+                const logNow = Date.now();
+                if (logNow - lastVolumeLogRef.current >= 500) {
+                  console.log('[RealtimeAudio] 🔍 Detecting...', sustainedDuration, '/', SPEECH_START_DURATION_MS, 'ms (volume:', volume.toFixed(4), ')');
+                  lastVolumeLogRef.current = logNow;
+                }
+              } else {
+                // Volume dropped before timer elapsed - false start
+                console.log('[RealtimeAudio] ❌ False start (volume dropped after', sustainedDuration, 'ms, needed', SPEECH_START_DURATION_MS, 'ms)');
+                speechStartRef.current = null;
+                speechConfirmedRef.current = false;
+                audioBufferRef.current = [];
+              }
+            } else if (speechDetected && !isSpeakingRef.current) {
+              // First volume detection - start timer and buffering
+              console.log('[RealtimeAudio] 🎤 Volume spike detected! Starting detection... (volume:', volume.toFixed(4), 'threshold:', VOLUME_THRESHOLD, ')');
+              speechStartRef.current = Date.now();
+              speechConfirmedRef.current = false;
+              audioBufferRef.current = [decodedChunk];
               silenceStartRef.current = null;
-              audioBufferRef.current = [audioData];
-              setIsSpeaking(true);
-              config.onSpeechStart?.();
             } else if (speechDetected && isSpeakingRef.current) {
-              // Continuing speech
-              audioBufferRef.current.push(audioData);
+              // Continuing confirmed speech - just buffer, no logs
+              audioBufferRef.current.push(decodedChunk);
               silenceStartRef.current = null;
             } else if (!speechDetected && isSpeakingRef.current) {
-              // Potential speech END
-              audioBufferRef.current.push(audioData);
+              // Potential speech END (after confirmed speech)
+              audioBufferRef.current.push(decodedChunk);
 
               if (silenceStartRef.current === null) {
                 silenceStartRef.current = Date.now();
+                console.log('[RealtimeAudio] 🔇 Silence detected, waiting', SILENCE_DURATION_MS, 'ms to confirm end...');
               } else {
                 const silenceDuration = Date.now() - silenceStartRef.current;
                 if (silenceDuration >= SILENCE_DURATION_MS) {
                   // Speech END confirmed
-                  console.log('[RealtimeAudio] Speech ended after', silenceDuration, 'ms silence');
+                  const totalChunks = audioBufferRef.current.length;
+                  console.log('[RealtimeAudio] 🛑 Speech ended after', silenceDuration, 'ms silence (captured', totalChunks, 'chunks)');
                   isSpeakingRef.current = false;
+                  speechStartRef.current = null;
+                  speechConfirmedRef.current = false;
                   silenceStartRef.current = null;
                   setIsSpeaking(false);
                   config.onSpeechEnd?.();
 
                   // Send buffered audio segment
                   if (audioBufferRef.current.length > 0) {
-                    const combinedAudio = audioBufferRef.current.join('');
-                    config.onAudioSegment?.(combinedAudio);
+                    // Concatenate all binary chunks
+                    const combinedBinary = concatenateUint8Arrays(audioBufferRef.current);
+                    const audioSizeKb = (combinedBinary.length / 1024).toFixed(2);
+                    console.log('[RealtimeAudio] 📤 Sending audio segment:', audioSizeKb, 'KB');
+                    // Convert to base64
+                    const combinedPcmBase64 = uint8ArrayToBase64(combinedBinary);
+                    // Convert PCM to WAV format
+                    const wavAudio = pcmToWav(combinedPcmBase64, 16000, 1, 16);
+                    config.onAudioSegment?.(wavAudio);
                     audioBufferRef.current = [];
                   }
                 }
@@ -165,13 +324,14 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
             }
           } else if (isSpeakingRef.current) {
             // Just buffer if we're speaking but not checking volume yet
-            audioBufferRef.current.push(audioData);
+            const decodedChunk = decodeBase64ToUint8Array(audioData);
+            audioBufferRef.current.push(decodedChunk);
           }
         },
       });
 
       setIsActive(true);
-      console.log('[RealtimeAudio] Realtime audio started successfully');
+      console.log('[RealtimeAudio] 🎙️ Audio capture started (threshold:', VOLUME_THRESHOLD, 'duration:', SPEECH_START_DURATION_MS, 'ms)');
     } catch (error) {
       console.error('[RealtimeAudio] Start error:', error);
       const err = error instanceof Error ? error : new Error(String(error));
@@ -198,6 +358,8 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
     // Reset state
     audioBufferRef.current = [];
     isSpeakingRef.current = false;
+    speechStartRef.current = null;
+    speechConfirmedRef.current = false;
     silenceStartRef.current = null;
     setIsActive(false);
     setIsSpeaking(false);