Prompt is too long

packages/voice-mobile/hooks/use-audio-player.ts

@@ -0,0 +1,259 @@
+import { useAudioPlayer as useExpoAudioPlayer, setAudioModeAsync } from 'expo-audio';
+import { Paths, File } from 'expo-file-system';
+import { useState, useRef, useEffect } from 'react';
+import { Platform } from 'react-native';
+
+interface QueuedAudio {
+  audioData: Blob;
+  resolve: (duration: number) => void;
+  reject: (error: Error) => void;
+}
+
+interface AudioPlayerOptions {
+  useSpeaker?: boolean;
+}
+
+/**
+ * Convert Blob to file URI for expo-audio playback
+ */
+async function blobToFile(blob: Blob): Promise<File> {
+  // Read blob as array buffer
+  const arrayBuffer = await blob.arrayBuffer();
+
+  // Create temporary file in cache directory
+  const fileName = `audio_${Date.now()}.mp3`;
+  const file = new File(Paths.cache, fileName);
+
+  // Write array buffer to file
+  const uint8Array = new Uint8Array(arrayBuffer);
+  file.create();
+  const stream = file.writableStream();
+  const writer = stream.getWriter();
+  await writer.write(uint8Array);
+  await writer.close();
+
+  return file;
+}
+
+/**
+ * Hook for audio playback with queue system matching web version functionality
+ */
+export function useAudioPlayer(options?: AudioPlayerOptions) {
+  const player = useExpoAudioPlayer();
+  const [isPlaying, setIsPlaying] = useState(false);
+  const [isPaused, setIsPaused] = useState(false);
+  const queueRef = useRef<QueuedAudio[]>([]);
+  const isProcessingQueueRef = useRef(false);
+  const currentRejectRef = useRef<((error: Error) => void) | null>(null);
+  const currentFileRef = useRef<File | null>(null);
+
+  // Configure audio mode based on speaker/earpiece preference
+  useEffect(() => {
+    async function configureAudioMode() {
+      try {
+        if (Platform.OS === 'android') {
+          // On Android, use shouldRouteThroughEarpiece
+          await setAudioModeAsync({
+            shouldRouteThroughEarpiece: !options?.useSpeaker,
+          });
+        } else if (Platform.OS === 'ios') {
+          // On iOS, the audio routing is controlled by the AVAudioSession category
+          // The default category should work, but we can configure it if needed
+          // For now, we rely on the iOS default behavior which routes to the receiver
+          // unless the user has changed the route (e.g., via speaker button in Control Center)
+        }
+      } catch (error) {
+        console.warn('[AudioPlayer] Failed to configure audio mode:', error);
+      }
+    }
+
+    configureAudioMode();
+  }, [options?.useSpeaker]);
+
+  // Monitor playback status
+  useEffect(() => {
+    if (!player.playing && isPlaying && !isPaused) {
+      // Playback finished
+      setIsPlaying(false);
+      processNextInQueue();
+    }
+  }, [player.playing, isPlaying, isPaused]);
+
+  async function play(audioData: Blob): Promise<number> {
+    return new Promise((resolve, reject) => {
+      // Add to queue with its promise handlers
+      queueRef.current.push({ audioData, resolve, reject });
+
+      // Start processing queue if not already processing
+      if (!isProcessingQueueRef.current) {
+        processQueue();
+      }
+    });
+  }
+
+  async function processQueue(): Promise<void> {
+    if (isProcessingQueueRef.current || queueRef.current.length === 0) {
+      return;
+    }
+
+    isProcessingQueueRef.current = true;
+
+    while (queueRef.current.length > 0) {
+      const item = queueRef.current.shift()!;
+      try {
+        const duration = await playAudio(item.audioData);
+        item.resolve(duration);
+      } catch (error) {
+        item.reject(error as Error);
+      }
+    }
+
+    isProcessingQueueRef.current = false;
+  }
+
+  async function processNextInQueue(): Promise<void> {
+    if (queueRef.current.length > 0) {
+      const item = queueRef.current.shift()!;
+      try {
+        const duration = await playAudio(item.audioData);
+        item.resolve(duration);
+      } catch (error) {
+        item.reject(error as Error);
+      }
+    } else {
+      isProcessingQueueRef.current = false;
+    }
+  }
+
+  async function playAudio(audioData: Blob): Promise<number> {
+    return new Promise(async (resolve, reject) => {
+      currentRejectRef.current = reject;
+
+      try {
+        console.log(
+          `[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
+        );
+
+        // Convert blob to file
+        const file = await blobToFile(audioData);
+        currentFileRef.current = file;
+
+        setIsPlaying(true);
+        setIsPaused(false);
+
+        // Load and play audio
+        await player.replace({ uri: file.uri });
+        player.play();
+
+        // Wait for playback to finish
+        const checkInterval = setInterval(() => {
+          if (!player.playing && player.duration > 0) {
+            clearInterval(checkInterval);
+
+            const duration = player.duration;
+            console.log(`[AudioPlayer] Playback finished (duration: ${duration}s)`);
+
+            setIsPlaying(false);
+            currentRejectRef.current = null;
+
+            // Clean up temporary file
+            if (currentFileRef.current) {
+              try {
+                currentFileRef.current.delete();
+              } catch (err: any) {
+                console.warn('[AudioPlayer] Failed to delete temp file:', err);
+              }
+              currentFileRef.current = null;
+            }
+
+            resolve(duration);
+          }
+        }, 100);
+
+      } catch (error) {
+        console.error('[AudioPlayer] Error playing audio:', error);
+        setIsPlaying(false);
+        currentRejectRef.current = null;
+
+        // Clean up temporary file
+        if (currentFileRef.current) {
+          try {
+            currentFileRef.current.delete();
+          } catch (err: any) {
+            console.warn('[AudioPlayer] Failed to delete temp file:', err);
+          }
+          currentFileRef.current = null;
+        }
+
+        reject(error);
+      }
+    });
+  }
+
+  function stop(): void {
+    // Stop currently playing audio
+    if (player) {
+      player.pause();
+    }
+    setIsPlaying(false);
+    setIsPaused(false);
+
+    // Clean up temporary file
+    if (currentFileRef.current) {
+      try {
+        currentFileRef.current.delete();
+      } catch (err: any) {
+        console.warn('[AudioPlayer] Failed to delete temp file:', err);
+      }
+      currentFileRef.current = null;
+    }
+
+    // Reject the current playing promise if it exists
+    if (currentRejectRef.current) {
+      currentRejectRef.current(new Error('Playback stopped'));
+      currentRejectRef.current = null;
+    }
+
+    // Reject all pending promises in the queue
+    while (queueRef.current.length > 0) {
+      const item = queueRef.current.shift()!;
+      item.reject(new Error('Playback stopped'));
+    }
+
+    isProcessingQueueRef.current = false;
+  }
+
+  function pause(): void {
+    if (player && isPlaying && !isPaused) {
+      player.pause();
+      setIsPaused(true);
+      console.log('[AudioPlayer] Playback paused');
+    }
+  }
+
+  function resume(): void {
+    if (player && isPaused) {
+      player.play();
+      setIsPaused(false);
+      console.log('[AudioPlayer] Playback resumed');
+    }
+  }
+
+  function clearQueue(): void {
+    // Reject all pending promises in the queue
+    while (queueRef.current.length > 0) {
+      const item = queueRef.current.shift()!;
+      item.reject(new Error('Queue cleared'));
+    }
+  }
+
+  return {
+    play,
+    stop,
+    pause,
+    resume,
+    isPlaying: () => isPlaying,
+    isPaused: () => isPaused,
+    clearQueue,
+  };
+}

packages/voice-mobile/hooks/use-audio-recorder.ts

@@ -0,0 +1,277 @@
+import { useAudioRecorder as useExpoAudioRecorder, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
+import { Paths, File, Directory, FileInfo } from 'expo-file-system';
+import { useState } from 'react';
+
+export interface AudioCaptureConfig {
+  sampleRate?: number;
+  numberOfChannels?: number;
+  bitRate?: number;
+}
+
+/**
+ * Workaround for Expo SDK 54 Android bug where audioRecorder.uri returns empty/zero-byte file
+ * https://github.com/expo/expo/issues/39646
+ */
+async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
+  try {
+    console.log('[AudioRecorder] Searching for recording file created at:', createdAt.toISOString());
+
+    const audioDir = new Directory(Paths.cache, 'Audio');
+    console.log('[AudioRecorder] Audio cache directory URI:', audioDir.uri);
+    console.log('[AudioRecorder] Directory exists:', audioDir.exists);
+
+    if (!audioDir.exists) {
+      console.log('[AudioRecorder] Audio cache directory does not exist');
+      return null;
+    }
+
+    const files = audioDir.list();
+    console.log('[AudioRecorder] Found files in Audio cache:', files.length);
+
+    if (!files.length) {
+      console.log('[AudioRecorder] No files found in Audio cache directory');
+      return null;
+    }
+
+    const validFiles = files
+      .map(file => {
+        const info = file.info();
+        console.log('[AudioRecorder] File info:', {
+          uri: info.uri,
+          size: info.size,
+          creationTime: info.creationTime ? new Date(info.creationTime).toISOString() : null,
+        });
+        return info;
+      })
+      .filter(f => f.size && f.size > 0);
+
+    console.log('[AudioRecorder] Valid files (size > 0):', validFiles.length);
+
+    if (validFiles.length === 0) {
+      console.log('[AudioRecorder] No valid files found (all are zero-byte)');
+      return null;
+    }
+
+    let closest: FileInfo | null = null;
+    let minDiff = Infinity;
+
+    for (const file of validFiles) {
+      if (!file.creationTime || !file.uri) continue;
+      const diff = Math.abs(file.creationTime - createdAt.getTime());
+      console.log('[AudioRecorder] Time diff for file:', {
+        uri: file.uri,
+        diffMs: diff,
+      });
+      if (diff < minDiff) {
+        closest = file;
+        minDiff = diff;
+      }
+    }
+
+    if (closest) {
+      const resultUri = closest.uri?.slice(0, -1) ?? null;
+      console.log('[AudioRecorder] Found closest file:', {
+        uri: resultUri,
+        size: closest.size,
+        timeDiffMs: minDiff,
+      });
+      return resultUri;
+    }
+
+    console.log('[AudioRecorder] No closest file found');
+    return null;
+  } catch (e) {
+    console.error('[AudioRecorder] Error finding actual recording file:', e);
+    return null;
+  }
+}
+
+/**
+ * Convert audio file URI to Blob for WebSocket transmission
+ * Returns a Blob-like object that works in React Native
+ */
+async function uriToBlob(uri: string): Promise<Blob> {
+  // Use expo-file-system to read the file
+  const file = new File(uri);
+  const base64 = await file.base64();
+  const size = file.size;
+
+  // React Native doesn't support creating Blobs from binary data
+  // Create a Blob-like object that has the methods we need
+  const blobLike = {
+    type: 'audio/m4a',
+    size: size,
+    arrayBuffer: async () => {
+      // Convert base64 to ArrayBuffer
+      const binaryString = atob(base64);
+      const bytes = new Uint8Array(binaryString.length);
+      for (let i = 0; i < binaryString.length; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+      }
+      return bytes.buffer;
+    },
+  } as Blob;
+
+  return blobLike;
+}
+
+/**
+ * Hook for audio recording with configuration matching web version
+ * Matches the web app's audio constraints:
+ * - 16000 sample rate (optimal for speech/Whisper)
+ * - 1 channel (mono)
+ * - Echo cancellation, noise suppression, auto gain control (voice_communication on Android)
+ */
+export function useAudioRecorder(config?: AudioCaptureConfig) {
+  const [isRecording, setIsRecording] = useState(false);
+  const [recordingStartTime, setRecordingStartTime] = useState<Date | null>(null);
+
+  // Create recording options matching web app constraints
+  const recordingOptions: RecordingOptions = {
+    ...RecordingPresets.HIGH_QUALITY,
+    sampleRate: config?.sampleRate || 16000,
+    numberOfChannels: config?.numberOfChannels || 1,
+    bitRate: config?.bitRate || 128000,
+    extension: '.m4a',
+    android: {
+      extension: '.m4a',
+      outputFormat: 'mpeg4',
+      audioEncoder: 'aac',
+      sampleRate: config?.sampleRate || 16000,
+      audioSource: 'voice_communication', // Enables echo cancellation, noise suppression, auto gain control
+    },
+    ios: {
+      extension: '.m4a',
+      audioQuality: 127, // High quality
+      sampleRate: config?.sampleRate || 16000,
+      linearPCMBitDepth: 16,
+      linearPCMIsBigEndian: false,
+      linearPCMIsFloat: false,
+    },
+    web: {
+      mimeType: 'audio/webm;codecs=opus',
+      bitsPerSecond: config?.bitRate || 128000,
+    },
+  };
+
+  const audioRecorder = useExpoAudioRecorder(recordingOptions);
+
+  async function start(): Promise<void> {
+    if (isRecording) {
+      throw new Error('Already recording');
+    }
+
+    try {
+      // Request microphone permissions
+      console.log('[AudioRecorder] Requesting recording permissions...');
+      const permissionResponse = await requestRecordingPermissionsAsync();
+
+      if (!permissionResponse.granted) {
+        throw new Error('Microphone permission denied. Please enable microphone access in your device settings.');
+      }
+
+      // Configure audio mode for recording
+      console.log('[AudioRecorder] Configuring audio mode...');
+      await setAudioModeAsync({
+        playsInSilentMode: true,
+        allowsRecording: true,
+      });
+
+      console.log('[AudioRecorder] Starting recording with options:', {
+        sampleRate: recordingOptions.sampleRate,
+        numberOfChannels: recordingOptions.numberOfChannels,
+        bitRate: recordingOptions.bitRate,
+      });
+
+      const startTime = new Date();
+      setRecordingStartTime(startTime);
+
+      // Prepare the recorder before recording (required step)
+      console.log('[AudioRecorder] Preparing recorder...');
+      await audioRecorder.prepareToRecordAsync();
+
+      console.log('[AudioRecorder] Starting recording...');
+      await audioRecorder.record();
+      setIsRecording(true);
+
+      console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
+      console.log('[AudioRecorder] Recorder isRecording:', audioRecorder.isRecording);
+    } catch (error: any) {
+      console.error('[AudioRecorder] Failed to start recording:', error);
+      throw new Error(`Failed to start audio recording: ${error.message}`);
+    }
+  }
+
+  async function stop(): Promise<Blob> {
+    if (!isRecording) {
+      throw new Error('Not recording');
+    }
+
+    try {
+      // Stop recording
+      await audioRecorder.stop();
+      setIsRecording(false);
+
+      // Get URI from recorder
+      let uri = audioRecorder.uri;
+      console.log('[AudioRecorder] Initial URI from recorder:', uri);
+
+      // Workaround for Expo SDK 54 Android bug - find actual recording file
+      if (recordingStartTime && (!uri || uri === '')) {
+        console.log('[AudioRecorder] Using workaround to find actual recording file...');
+        const actualUri = await getActualRecordingUri(recordingStartTime);
+        if (actualUri) {
+          uri = actualUri;
+          console.log('[AudioRecorder] Found actual recording URI:', uri);
+        }
+      }
+
+      if (!uri || uri === '') {
+        throw new Error('No recording URI found');
+      }
+
+      // Get file info
+      const file = new File(uri);
+      const exists = file.exists;
+      console.log('[AudioRecorder] File exists:', exists);
+
+      if (!exists) {
+        throw new Error('Recording file does not exist');
+      }
+
+      // Convert URI to Blob
+      const audioBlob = await uriToBlob(uri);
+
+      console.log('[AudioRecorder] Recording converted to blob:', {
+        size: audioBlob.size,
+        type: audioBlob.type,
+      });
+
+      // Clean up the temporary file
+      file.delete();
+
+      // Reset start time
+      setRecordingStartTime(null);
+
+      return audioBlob;
+    } catch (error: any) {
+      setIsRecording(false);
+      setRecordingStartTime(null);
+      console.error('[AudioRecorder] Failed to stop recording:', error);
+      throw new Error(`Failed to stop audio recording: ${error.message}`);
+    }
+  }
+
+  function getSupportedMimeType(): string | null {
+    // On native platforms, expo-audio uses m4a/AAC
+    // On web, it can use webm/opus
+    return 'audio/m4a';
+  }
+
+  return {
+    start,
+    stop,
+    isRecording: () => isRecording,
+    getSupportedMimeType,
+  };
+}

packages/voice-mobile/hooks/use-settings.ts

@@ -0,0 +1,78 @@
+import { useState, useEffect, useCallback } from 'react';
+import AsyncStorage from '@react-native-async-storage/async-storage';
+
+const STORAGE_KEY = '@voice-assistant:settings';
+
+export interface Settings {
+  serverUrl: string;
+  useSpeaker: boolean;
+  keepScreenOn: boolean;
+  theme: 'dark' | 'light' | 'auto';
+}
+
+const DEFAULT_SETTINGS: Settings = {
+  serverUrl: 'wss://mohameds-macbook-pro.tail8fe838.ts.net/ws',
+  useSpeaker: true,
+  keepScreenOn: true,
+  theme: 'dark',
+};
+
+export interface UseSettingsReturn {
+  settings: Settings;
+  isLoading: boolean;
+  updateSettings: (updates: Partial<Settings>) => Promise<void>;
+  resetSettings: () => Promise<void>;
+}
+
+export function useSettings(): UseSettingsReturn {
+  const [settings, setSettings] = useState<Settings>(DEFAULT_SETTINGS);
+  const [isLoading, setIsLoading] = useState(true);
+
+  // Load settings from AsyncStorage on mount
+  useEffect(() => {
+    loadSettings();
+  }, []);
+
+  async function loadSettings() {
+    try {
+      const stored = await AsyncStorage.getItem(STORAGE_KEY);
+      if (stored) {
+        const parsed = JSON.parse(stored) as Partial<Settings>;
+        setSettings({ ...DEFAULT_SETTINGS, ...parsed });
+      }
+    } catch (error) {
+      console.error('[Settings] Failed to load settings:', error);
+      // Continue with default settings
+    } finally {
+      setIsLoading(false);
+    }
+  }
+
+  const updateSettings = useCallback(async (updates: Partial<Settings>) => {
+    try {
+      const newSettings = { ...settings, ...updates };
+      setSettings(newSettings);
+      await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(newSettings));
+    } catch (error) {
+      console.error('[Settings] Failed to save settings:', error);
+      throw error;
+    }
+  }, [settings]);
+
+  const resetSettings = useCallback(async () => {
+    try {
+      setSettings(DEFAULT_SETTINGS);
+      await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(DEFAULT_SETTINGS));
+    } catch (error) {
+      console.error('[Settings] Failed to reset settings:', error);
+      throw error;
+    }
+  }, []);
+
+  return {
+    settings,
+    isLoading,
+    updateSettings,
+    resetSettings,
+  };
+}

packages/voice-mobile/hooks/use-websocket.ts

@@ -0,0 +1,161 @@
+import { useEffect, useRef, useState, useCallback } from 'react';
+import type {
+  WSInboundMessage,
+  WSOutboundMessage,
+  SessionOutboundMessage
+} from '@voice-assistant/server/messages';
+
+export interface UseWebSocketReturn {
+  isConnected: boolean;
+  conversationId: string | null;
+  send: (message: WSInboundMessage) => void;
+  on: (type: SessionOutboundMessage['type'], handler: (message: SessionOutboundMessage) => void) => () => void;
+  sendPing: () => void;
+  sendUserMessage: (message: string) => void;
+}
+
+export function useWebSocket(url: string, conversationId?: string | null): UseWebSocketReturn {
+  const [isConnected, setIsConnected] = useState(false);
+  const [currentConversationId, setCurrentConversationId] = useState<string | null>(null);
+  const wsRef = useRef<WebSocket | null>(null);
+  const handlersRef = useRef<Map<SessionOutboundMessage['type'], Set<(message: SessionOutboundMessage) => void>>>(new Map());
+  const reconnectTimeoutRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
+
+  const connect = useCallback(() => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      return;
+    }
+
+    try {
+      // Add conversation ID to URL if provided
+      const wsUrl = conversationId ? `${url}?conversationId=${conversationId}` : url;
+      const ws = new WebSocket(wsUrl);
+
+      ws.onopen = () => {
+        console.log('[WS] Connected to server');
+        setIsConnected(true);
+      };
+
+      ws.onclose = () => {
+        console.log('[WS] Disconnected from server');
+        setIsConnected(false);
+
+        // Attempt to reconnect after 3 seconds
+        reconnectTimeoutRef.current = setTimeout(() => {
+          console.log('[WS] Attempting to reconnect...');
+          connect();
+        }, 3000);
+      };
+
+      ws.onerror = (error) => {
+        console.error('[WS] Error:', error);
+      };
+
+      ws.onmessage = (event) => {
+        try {
+          const wsMessage: WSOutboundMessage = JSON.parse(event.data);
+
+          // Only session messages trigger handlers
+          if (wsMessage.type === 'session') {
+            const sessionMessage = wsMessage.message;
+            console.log(`[WS] Received session message type: ${sessionMessage.type}`);
+
+            // Track conversation ID when loaded
+            if (sessionMessage.type === 'conversation_loaded') {
+              setCurrentConversationId(sessionMessage.payload.conversationId);
+            }
+
+            // Call all registered handlers for this message type
+            const handlers = handlersRef.current.get(sessionMessage.type);
+            if (handlers) {
+              handlers.forEach((handler) => {
+                try {
+                  handler(sessionMessage);
+                } catch (err) {
+                  console.error(`[WS] Error in handler for ${sessionMessage.type}:`, err);
+                }
+              });
+            }
+          } else {
+            // pong - just log
+            console.log(`[WS] Received ${wsMessage.type}`);
+          }
+        } catch (err) {
+          console.error('[WS] Failed to parse message:', err);
+        }
+      };
+
+      wsRef.current = ws;
+    } catch (err) {
+      console.error('[WS] Failed to create WebSocket:', err);
+    }
+  }, [url, conversationId]);
+
+  useEffect(() => {
+    connect();
+
+    return () => {
+      if (reconnectTimeoutRef.current) {
+        clearTimeout(reconnectTimeoutRef.current);
+      }
+      if (wsRef.current) {
+        wsRef.current.close();
+      }
+    };
+  }, [connect]);
+
+  const send = useCallback((message: WSInboundMessage) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify(message));
+    } else {
+      console.warn('[WS] Cannot send message - not connected');
+    }
+  }, []);
+
+  const on = useCallback((
+    type: SessionOutboundMessage['type'],
+    handler: (message: SessionOutboundMessage) => void
+  ) => {
+    if (!handlersRef.current.has(type)) {
+      handlersRef.current.set(type, new Set());
+    }
+    handlersRef.current.get(type)!.add(handler);
+
+    // Return cleanup function
+    return () => {
+      const handlers = handlersRef.current.get(type);
+      if (handlers) {
+        handlers.delete(handler);
+        if (handlers.size === 0) {
+          handlersRef.current.delete(type);
+        }
+      }
+    };
+  }, []);
+
+  const sendPing = useCallback(() => {
+    send({ type: 'ping' });
+  }, [send]);
+
+  const sendUserMessage = useCallback(
+    (message: string) => {
+      send({
+        type: 'session',
+        message: {
+          type: 'user_text',
+          text: message,
+        },
+      });
+    },
+    [send]
+  );
+
+  return {
+    isConnected,
+    conversationId: currentConversationId,
+    send,
+    on,
+    sendPing,
+    sendUserMessage,
+  };
+}