Prompt is too long
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,259 @@
|
||||
import { useAudioPlayer as useExpoAudioPlayer, setAudioModeAsync } from 'expo-audio';
|
||||
import { Paths, File } from 'expo-file-system';
|
||||
import { useState, useRef, useEffect } from 'react';
|
||||
import { Platform } from 'react-native';
|
||||
|
||||
interface QueuedAudio {
|
||||
audioData: Blob;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
interface AudioPlayerOptions {
|
||||
useSpeaker?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Blob to file URI for expo-audio playback
|
||||
*/
|
||||
async function blobToFile(blob: Blob): Promise<File> {
|
||||
// Read blob as array buffer
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
|
||||
// Create temporary file in cache directory
|
||||
const fileName = `audio_${Date.now()}.mp3`;
|
||||
const file = new File(Paths.cache, fileName);
|
||||
|
||||
// Write array buffer to file
|
||||
const uint8Array = new Uint8Array(arrayBuffer);
|
||||
file.create();
|
||||
const stream = file.writableStream();
|
||||
const writer = stream.getWriter();
|
||||
await writer.write(uint8Array);
|
||||
await writer.close();
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio playback with queue system matching web version functionality
|
||||
*/
|
||||
export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
const player = useExpoAudioPlayer();
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [isPaused, setIsPaused] = useState(false);
|
||||
const queueRef = useRef<QueuedAudio[]>([]);
|
||||
const isProcessingQueueRef = useRef(false);
|
||||
const currentRejectRef = useRef<((error: Error) => void) | null>(null);
|
||||
const currentFileRef = useRef<File | null>(null);
|
||||
|
||||
// Configure audio mode based on speaker/earpiece preference
|
||||
useEffect(() => {
|
||||
async function configureAudioMode() {
|
||||
try {
|
||||
if (Platform.OS === 'android') {
|
||||
// On Android, use shouldRouteThroughEarpiece
|
||||
await setAudioModeAsync({
|
||||
shouldRouteThroughEarpiece: !options?.useSpeaker,
|
||||
});
|
||||
} else if (Platform.OS === 'ios') {
|
||||
// On iOS, the audio routing is controlled by the AVAudioSession category
|
||||
// The default category should work, but we can configure it if needed
|
||||
// For now, we rely on the iOS default behavior which routes to the receiver
|
||||
// unless the user has changed the route (e.g., via speaker button in Control Center)
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('[AudioPlayer] Failed to configure audio mode:', error);
|
||||
}
|
||||
}
|
||||
|
||||
configureAudioMode();
|
||||
}, [options?.useSpeaker]);
|
||||
|
||||
// Monitor playback status
|
||||
useEffect(() => {
|
||||
if (!player.playing && isPlaying && !isPaused) {
|
||||
// Playback finished
|
||||
setIsPlaying(false);
|
||||
processNextInQueue();
|
||||
}
|
||||
}, [player.playing, isPlaying, isPaused]);
|
||||
|
||||
async function play(audioData: Blob): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Add to queue with its promise handlers
|
||||
queueRef.current.push({ audioData, resolve, reject });
|
||||
|
||||
// Start processing queue if not already processing
|
||||
if (!isProcessingQueueRef.current) {
|
||||
processQueue();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = true;
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
async function processNextInQueue(): Promise<void> {
|
||||
if (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
} else {
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audioData: Blob): Promise<number> {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
currentRejectRef.current = reject;
|
||||
|
||||
try {
|
||||
console.log(
|
||||
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
|
||||
);
|
||||
|
||||
// Convert blob to file
|
||||
const file = await blobToFile(audioData);
|
||||
currentFileRef.current = file;
|
||||
|
||||
setIsPlaying(true);
|
||||
setIsPaused(false);
|
||||
|
||||
// Load and play audio
|
||||
await player.replace({ uri: file.uri });
|
||||
player.play();
|
||||
|
||||
// Wait for playback to finish
|
||||
const checkInterval = setInterval(() => {
|
||||
if (!player.playing && player.duration > 0) {
|
||||
clearInterval(checkInterval);
|
||||
|
||||
const duration = player.duration;
|
||||
console.log(`[AudioPlayer] Playback finished (duration: ${duration}s)`);
|
||||
|
||||
setIsPlaying(false);
|
||||
currentRejectRef.current = null;
|
||||
|
||||
// Clean up temporary file
|
||||
if (currentFileRef.current) {
|
||||
try {
|
||||
currentFileRef.current.delete();
|
||||
} catch (err: any) {
|
||||
console.warn('[AudioPlayer] Failed to delete temp file:', err);
|
||||
}
|
||||
currentFileRef.current = null;
|
||||
}
|
||||
|
||||
resolve(duration);
|
||||
}
|
||||
}, 100);
|
||||
|
||||
} catch (error) {
|
||||
console.error('[AudioPlayer] Error playing audio:', error);
|
||||
setIsPlaying(false);
|
||||
currentRejectRef.current = null;
|
||||
|
||||
// Clean up temporary file
|
||||
if (currentFileRef.current) {
|
||||
try {
|
||||
currentFileRef.current.delete();
|
||||
} catch (err: any) {
|
||||
console.warn('[AudioPlayer] Failed to delete temp file:', err);
|
||||
}
|
||||
currentFileRef.current = null;
|
||||
}
|
||||
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
// Stop currently playing audio
|
||||
if (player) {
|
||||
player.pause();
|
||||
}
|
||||
setIsPlaying(false);
|
||||
setIsPaused(false);
|
||||
|
||||
// Clean up temporary file
|
||||
if (currentFileRef.current) {
|
||||
try {
|
||||
currentFileRef.current.delete();
|
||||
} catch (err: any) {
|
||||
console.warn('[AudioPlayer] Failed to delete temp file:', err);
|
||||
}
|
||||
currentFileRef.current = null;
|
||||
}
|
||||
|
||||
// Reject the current playing promise if it exists
|
||||
if (currentRejectRef.current) {
|
||||
currentRejectRef.current(new Error('Playback stopped'));
|
||||
currentRejectRef.current = null;
|
||||
}
|
||||
|
||||
// Reject all pending promises in the queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error('Playback stopped'));
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
function pause(): void {
|
||||
if (player && isPlaying && !isPaused) {
|
||||
player.pause();
|
||||
setIsPaused(true);
|
||||
console.log('[AudioPlayer] Playback paused');
|
||||
}
|
||||
}
|
||||
|
||||
function resume(): void {
|
||||
if (player && isPaused) {
|
||||
player.play();
|
||||
setIsPaused(false);
|
||||
console.log('[AudioPlayer] Playback resumed');
|
||||
}
|
||||
}
|
||||
|
||||
function clearQueue(): void {
|
||||
// Reject all pending promises in the queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error('Queue cleared'));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
play,
|
||||
stop,
|
||||
pause,
|
||||
resume,
|
||||
isPlaying: () => isPlaying,
|
||||
isPaused: () => isPaused,
|
||||
clearQueue,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
import { useAudioRecorder as useExpoAudioRecorder, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
|
||||
import { Paths, File, Directory, FileInfo } from 'expo-file-system';
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
export interface AudioCaptureConfig {
|
||||
sampleRate?: number;
|
||||
numberOfChannels?: number;
|
||||
bitRate?: number;
|
||||
onAudioLevel?: (level: number) => void;
|
||||
onSpeechSegment?: (audioBlob: Blob) => void;
|
||||
enableContinuousRecording?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Workaround for Expo SDK 54 Android bug where audioRecorder.uri returns empty/zero-byte file
|
||||
* https://github.com/expo/expo/issues/39646
|
||||
*/
|
||||
async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
|
||||
try {
|
||||
console.log('[AudioRecorder] Searching for recording file created at:', createdAt.toISOString());
|
||||
|
||||
const audioDir = new Directory(Paths.cache, 'Audio');
|
||||
console.log('[AudioRecorder] Audio cache directory URI:', audioDir.uri);
|
||||
console.log('[AudioRecorder] Directory exists:', audioDir.exists);
|
||||
|
||||
if (!audioDir.exists) {
|
||||
console.log('[AudioRecorder] Audio cache directory does not exist');
|
||||
return null;
|
||||
}
|
||||
|
||||
const files = audioDir.list();
|
||||
console.log('[AudioRecorder] Found files in Audio cache:', files.length);
|
||||
|
||||
if (!files.length) {
|
||||
console.log('[AudioRecorder] No files found in Audio cache directory');
|
||||
return null;
|
||||
}
|
||||
|
||||
const validFiles = files
|
||||
.map(file => {
|
||||
const info = file.info();
|
||||
console.log('[AudioRecorder] File info:', {
|
||||
uri: info.uri,
|
||||
size: info.size,
|
||||
creationTime: info.creationTime ? new Date(info.creationTime).toISOString() : null,
|
||||
});
|
||||
return info;
|
||||
})
|
||||
.filter(f => f.size && f.size > 0);
|
||||
|
||||
console.log('[AudioRecorder] Valid files (size > 0):', validFiles.length);
|
||||
|
||||
if (validFiles.length === 0) {
|
||||
console.log('[AudioRecorder] No valid files found (all are zero-byte)');
|
||||
return null;
|
||||
}
|
||||
|
||||
let closest: FileInfo | null = null;
|
||||
let minDiff = Infinity;
|
||||
|
||||
for (const file of validFiles) {
|
||||
if (!file.creationTime || !file.uri) continue;
|
||||
const diff = Math.abs(file.creationTime - createdAt.getTime());
|
||||
console.log('[AudioRecorder] Time diff for file:', {
|
||||
uri: file.uri,
|
||||
diffMs: diff,
|
||||
});
|
||||
if (diff < minDiff) {
|
||||
closest = file;
|
||||
minDiff = diff;
|
||||
}
|
||||
}
|
||||
|
||||
if (closest) {
|
||||
const resultUri = closest.uri?.slice(0, -1) ?? null;
|
||||
console.log('[AudioRecorder] Found closest file:', {
|
||||
uri: resultUri,
|
||||
size: closest.size,
|
||||
timeDiffMs: minDiff,
|
||||
});
|
||||
return resultUri;
|
||||
}
|
||||
|
||||
console.log('[AudioRecorder] No closest file found');
|
||||
return null;
|
||||
} catch (e) {
|
||||
console.error('[AudioRecorder] Error finding actual recording file:', e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert audio file URI to Blob for WebSocket transmission
|
||||
* Returns a Blob-like object that works in React Native
|
||||
*/
|
||||
async function uriToBlob(uri: string): Promise<Blob> {
|
||||
// Use expo-file-system to read the file
|
||||
const file = new File(uri);
|
||||
const base64 = await file.base64();
|
||||
const size = file.size;
|
||||
|
||||
// React Native doesn't support creating Blobs from binary data
|
||||
// Create a Blob-like object that has the methods we need
|
||||
const blobLike = {
|
||||
type: 'audio/m4a',
|
||||
size: size,
|
||||
arrayBuffer: async () => {
|
||||
// Convert base64 to ArrayBuffer
|
||||
const binaryString = atob(base64);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
return bytes.buffer;
|
||||
},
|
||||
} as Blob;
|
||||
|
||||
return blobLike;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio recording with configuration matching web version
|
||||
* Matches the web app's audio constraints:
|
||||
* - 16000 sample rate (optimal for speech/Whisper)
|
||||
* - 1 channel (mono)
|
||||
* - Echo cancellation, noise suppression, auto gain control (voice_communication on Android)
|
||||
*/
|
||||
export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [recordingStartTime, setRecordingStartTime] = useState<Date | null>(null);
|
||||
|
||||
// Create recording options matching web app constraints
|
||||
const recordingOptions: RecordingOptions = {
|
||||
...RecordingPresets.HIGH_QUALITY,
|
||||
sampleRate: config?.sampleRate || 16000,
|
||||
numberOfChannels: config?.numberOfChannels || 1,
|
||||
bitRate: config?.bitRate || 128000,
|
||||
extension: '.m4a',
|
||||
isMeteringEnabled: !!config?.onAudioLevel, // Enable metering if callback provided
|
||||
android: {
|
||||
extension: '.m4a',
|
||||
outputFormat: 'mpeg4',
|
||||
audioEncoder: 'aac',
|
||||
sampleRate: config?.sampleRate || 16000,
|
||||
audioSource: 'voice_communication', // Enables echo cancellation, noise suppression, auto gain control
|
||||
},
|
||||
ios: {
|
||||
extension: '.m4a',
|
||||
audioQuality: 127, // High quality
|
||||
sampleRate: config?.sampleRate || 16000,
|
||||
linearPCMBitDepth: 16,
|
||||
linearPCMIsBigEndian: false,
|
||||
linearPCMIsFloat: false,
|
||||
},
|
||||
web: {
|
||||
mimeType: 'audio/webm;codecs=opus',
|
||||
bitsPerSecond: config?.bitRate || 128000,
|
||||
},
|
||||
};
|
||||
|
||||
const audioRecorder = useExpoAudioRecorder(recordingOptions);
|
||||
|
||||
// Monitor audio levels if metering is enabled
|
||||
useEffect(() => {
|
||||
if (!config?.onAudioLevel || !isRecording) return;
|
||||
|
||||
const interval = setInterval(() => {
|
||||
const metering = audioRecorder.state?.metering;
|
||||
if (metering !== undefined && metering !== null) {
|
||||
// Normalize metering value (typically ranges from -160 to 0 dB)
|
||||
// Convert to 0-1 range where 0 is silence and 1 is loud
|
||||
// We'll use -40 dB as the threshold for "loud"
|
||||
const normalized = Math.max(0, Math.min(1, (metering + 40) / 40));
|
||||
config.onAudioLevel(normalized);
|
||||
}
|
||||
}, 100); // Check every 100ms
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [isRecording, config, audioRecorder.state?.metering]);
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (isRecording) {
|
||||
throw new Error('Already recording');
|
||||
}
|
||||
|
||||
try {
|
||||
// Request microphone permissions
|
||||
console.log('[AudioRecorder] Requesting recording permissions...');
|
||||
const permissionResponse = await requestRecordingPermissionsAsync();
|
||||
|
||||
if (!permissionResponse.granted) {
|
||||
throw new Error('Microphone permission denied. Please enable microphone access in your device settings.');
|
||||
}
|
||||
|
||||
// Configure audio mode for recording
|
||||
console.log('[AudioRecorder] Configuring audio mode...');
|
||||
await setAudioModeAsync({
|
||||
playsInSilentMode: true,
|
||||
allowsRecording: true,
|
||||
});
|
||||
|
||||
console.log('[AudioRecorder] Starting recording with options:', {
|
||||
sampleRate: recordingOptions.sampleRate,
|
||||
numberOfChannels: recordingOptions.numberOfChannels,
|
||||
bitRate: recordingOptions.bitRate,
|
||||
});
|
||||
|
||||
const startTime = new Date();
|
||||
setRecordingStartTime(startTime);
|
||||
|
||||
// Prepare the recorder before recording (required step)
|
||||
console.log('[AudioRecorder] Preparing recorder...');
|
||||
await audioRecorder.prepareToRecordAsync();
|
||||
|
||||
console.log('[AudioRecorder] Starting recording...');
|
||||
await audioRecorder.record();
|
||||
setIsRecording(true);
|
||||
|
||||
console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
|
||||
console.log('[AudioRecorder] Recorder isRecording:', audioRecorder.isRecording);
|
||||
} catch (error: any) {
|
||||
console.error('[AudioRecorder] Failed to start recording:', error);
|
||||
throw new Error(`Failed to start audio recording: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<Blob> {
|
||||
if (!isRecording) {
|
||||
throw new Error('Not recording');
|
||||
}
|
||||
|
||||
try {
|
||||
// Stop recording
|
||||
await audioRecorder.stop();
|
||||
setIsRecording(false);
|
||||
|
||||
// Get URI from recorder
|
||||
let uri = audioRecorder.uri;
|
||||
console.log('[AudioRecorder] Initial URI from recorder:', uri);
|
||||
|
||||
// Workaround for Expo SDK 54 Android bug - find actual recording file
|
||||
if (recordingStartTime && (!uri || uri === '')) {
|
||||
console.log('[AudioRecorder] Using workaround to find actual recording file...');
|
||||
const actualUri = await getActualRecordingUri(recordingStartTime);
|
||||
if (actualUri) {
|
||||
uri = actualUri;
|
||||
console.log('[AudioRecorder] Found actual recording URI:', uri);
|
||||
}
|
||||
}
|
||||
|
||||
if (!uri || uri === '') {
|
||||
throw new Error('No recording URI found');
|
||||
}
|
||||
|
||||
// Get file info
|
||||
const file = new File(uri);
|
||||
const exists = file.exists;
|
||||
console.log('[AudioRecorder] File exists:', exists);
|
||||
|
||||
if (!exists) {
|
||||
throw new Error('Recording file does not exist');
|
||||
}
|
||||
|
||||
// Convert URI to Blob
|
||||
const audioBlob = await uriToBlob(uri);
|
||||
|
||||
console.log('[AudioRecorder] Recording converted to blob:', {
|
||||
size: audioBlob.size,
|
||||
type: audioBlob.type,
|
||||
});
|
||||
|
||||
// Clean up the temporary file
|
||||
file.delete();
|
||||
|
||||
// Reset start time
|
||||
setRecordingStartTime(null);
|
||||
|
||||
return audioBlob;
|
||||
} catch (error: any) {
|
||||
setIsRecording(false);
|
||||
setRecordingStartTime(null);
|
||||
console.error('[AudioRecorder] Failed to stop recording:', error);
|
||||
throw new Error(`Failed to stop audio recording: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function getSupportedMimeType(): string | null {
|
||||
// On native platforms, expo-audio uses m4a/AAC
|
||||
// On web, it can use webm/opus
|
||||
return 'audio/m4a';
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
isRecording: () => isRecording,
|
||||
getSupportedMimeType,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { useColorScheme } from 'react-native';
|
||||
@@ -0,0 +1,21 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useColorScheme as useRNColorScheme } from 'react-native';
|
||||
|
||||
/**
|
||||
* To support static rendering, this value needs to be re-calculated on the client side for web
|
||||
*/
|
||||
export function useColorScheme() {
|
||||
const [hasHydrated, setHasHydrated] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
setHasHydrated(true);
|
||||
}, []);
|
||||
|
||||
const colorScheme = useRNColorScheme();
|
||||
|
||||
if (hasHydrated) {
|
||||
return colorScheme;
|
||||
}
|
||||
|
||||
return 'light';
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import { Platform } from 'react-native';
|
||||
import { useAudioRecorder } from '@siteed/expo-audio-studio';
|
||||
import { check, request, PERMISSIONS, RESULTS } from 'react-native-permissions';
|
||||
|
||||
export interface RealtimeAudioConfig {
|
||||
onAudioSegment?: (audioData: string) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
volumeThreshold?: number; // RMS threshold for speech detection (0-1)
|
||||
silenceDuration?: number; // ms of silence before ending segment
|
||||
}
|
||||
|
||||
export interface RealtimeAudio {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
isActive: boolean;
|
||||
isSpeaking: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for realtime audio streaming with volume-based VAD
|
||||
* Uses expo-audio-studio for recording with echo cancellation and noise suppression
|
||||
*/
|
||||
export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
|
||||
const [isActive, setIsActive] = useState(false);
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
|
||||
const audioRecorder = useAudioRecorder();
|
||||
const audioBufferRef = useRef<string[]>([]);
|
||||
const silenceStartRef = useRef<number | null>(null);
|
||||
const isSpeakingRef = useRef(false);
|
||||
const lastVolumeCheckRef = useRef<number>(Date.now());
|
||||
|
||||
const VOLUME_THRESHOLD = config.volumeThreshold ?? 0.02;
|
||||
const SILENCE_DURATION_MS = config.silenceDuration ?? 1000;
|
||||
const VOLUME_CHECK_INTERVAL = 200; // Check volume every 200ms
|
||||
|
||||
async function requestMicrophonePermission(): Promise<boolean> {
|
||||
try {
|
||||
const permission = Platform.OS === 'ios'
|
||||
? PERMISSIONS.IOS.MICROPHONE
|
||||
: PERMISSIONS.ANDROID.RECORD_AUDIO;
|
||||
|
||||
const status = await check(permission);
|
||||
if (status === RESULTS.GRANTED) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const result = await request(permission);
|
||||
return result === RESULTS.GRANTED;
|
||||
} catch (error) {
|
||||
console.error('[RealtimeAudio] Permission error:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function calculateVolume(base64Audio: string): number {
|
||||
try {
|
||||
// Decode base64 to binary
|
||||
const binary = atob(base64Audio);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
|
||||
// Calculate RMS from PCM data (assuming 16-bit PCM)
|
||||
let sum = 0;
|
||||
for (let i = 0; i < bytes.length - 1; i += 2) {
|
||||
const sample = (bytes[i] | (bytes[i + 1] << 8)) / 32768.0; // Convert to -1 to 1
|
||||
sum += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(sum / (bytes.length / 2));
|
||||
return rms;
|
||||
} catch (error) {
|
||||
console.error('[RealtimeAudio] Volume calculation error:', error);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (isActive) {
|
||||
console.log('[RealtimeAudio] Already active');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('[RealtimeAudio] Starting realtime audio...');
|
||||
|
||||
// Request permissions
|
||||
const hasPermission = await requestMicrophonePermission();
|
||||
if (!hasPermission) {
|
||||
throw new Error('Microphone permission denied');
|
||||
}
|
||||
|
||||
// Start recording with audio streaming
|
||||
await audioRecorder.startRecording({
|
||||
sampleRate: 16000,
|
||||
channels: 1,
|
||||
encoding: 'pcm_16bit',
|
||||
interval: 100, // Get chunks every 100ms
|
||||
onAudioStream: async (event: any) => {
|
||||
if (!event.data) return;
|
||||
|
||||
const audioData = event.data; // base64 PCM data
|
||||
|
||||
// Check volume periodically
|
||||
const now = Date.now();
|
||||
if (now - lastVolumeCheckRef.current >= VOLUME_CHECK_INTERVAL) {
|
||||
lastVolumeCheckRef.current = now;
|
||||
|
||||
const volume = calculateVolume(audioData);
|
||||
const speechDetected = volume > VOLUME_THRESHOLD;
|
||||
|
||||
if (speechDetected && !isSpeakingRef.current) {
|
||||
// Speech START
|
||||
console.log('[RealtimeAudio] Speech started (volume:', volume.toFixed(4), ')');
|
||||
isSpeakingRef.current = true;
|
||||
silenceStartRef.current = null;
|
||||
audioBufferRef.current = [audioData];
|
||||
setIsSpeaking(true);
|
||||
config.onSpeechStart?.();
|
||||
} else if (speechDetected && isSpeakingRef.current) {
|
||||
// Continuing speech
|
||||
audioBufferRef.current.push(audioData);
|
||||
silenceStartRef.current = null;
|
||||
} else if (!speechDetected && isSpeakingRef.current) {
|
||||
// Potential speech END
|
||||
audioBufferRef.current.push(audioData);
|
||||
|
||||
if (silenceStartRef.current === null) {
|
||||
silenceStartRef.current = Date.now();
|
||||
} else {
|
||||
const silenceDuration = Date.now() - silenceStartRef.current;
|
||||
if (silenceDuration >= SILENCE_DURATION_MS) {
|
||||
// Speech END confirmed
|
||||
console.log('[RealtimeAudio] Speech ended after', silenceDuration, 'ms silence');
|
||||
isSpeakingRef.current = false;
|
||||
silenceStartRef.current = null;
|
||||
setIsSpeaking(false);
|
||||
config.onSpeechEnd?.();
|
||||
|
||||
// Send buffered audio segment
|
||||
if (audioBufferRef.current.length > 0) {
|
||||
const combinedAudio = audioBufferRef.current.join('');
|
||||
config.onAudioSegment?.(combinedAudio);
|
||||
audioBufferRef.current = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (isSpeakingRef.current) {
|
||||
// Just buffer if we're speaking but not checking volume yet
|
||||
audioBufferRef.current.push(audioData);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
setIsActive(true);
|
||||
console.log('[RealtimeAudio] Realtime audio started successfully');
|
||||
} catch (error) {
|
||||
console.error('[RealtimeAudio] Start error:', error);
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
|
||||
// Cleanup on error
|
||||
await stop();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<void> {
|
||||
console.log('[RealtimeAudio] Stopping realtime audio...');
|
||||
|
||||
// Stop recording
|
||||
if (audioRecorder.isRecording) {
|
||||
try {
|
||||
await audioRecorder.stopRecording();
|
||||
} catch (error) {
|
||||
console.error('[RealtimeAudio] Error stopping recording:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Reset state
|
||||
audioBufferRef.current = [];
|
||||
isSpeakingRef.current = false;
|
||||
silenceStartRef.current = null;
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
|
||||
console.log('[RealtimeAudio] Realtime audio stopped');
|
||||
}
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (isActive) {
|
||||
stop();
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
isActive,
|
||||
isSpeaking,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import AsyncStorage from '@react-native-async-storage/async-storage';
|
||||
|
||||
const STORAGE_KEY = '@voice-dev:settings';
|
||||
|
||||
export interface Settings {
|
||||
serverUrl: string;
|
||||
useSpeaker: boolean;
|
||||
keepScreenOn: boolean;
|
||||
theme: 'dark' | 'light' | 'auto';
|
||||
}
|
||||
|
||||
const DEFAULT_SETTINGS: Settings = {
|
||||
serverUrl: 'wss://mohameds-macbook-pro.tail8fe838.ts.net/ws',
|
||||
useSpeaker: true,
|
||||
keepScreenOn: true,
|
||||
theme: 'dark',
|
||||
};
|
||||
|
||||
export interface UseSettingsReturn {
|
||||
settings: Settings;
|
||||
isLoading: boolean;
|
||||
updateSettings: (updates: Partial<Settings>) => Promise<void>;
|
||||
resetSettings: () => Promise<void>;
|
||||
}
|
||||
|
||||
export function useSettings(): UseSettingsReturn {
|
||||
const [settings, setSettings] = useState<Settings>(DEFAULT_SETTINGS);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
|
||||
// Load settings from AsyncStorage on mount
|
||||
useEffect(() => {
|
||||
loadSettings();
|
||||
}, []);
|
||||
|
||||
async function loadSettings() {
|
||||
try {
|
||||
const stored = await AsyncStorage.getItem(STORAGE_KEY);
|
||||
if (stored) {
|
||||
const parsed = JSON.parse(stored) as Partial<Settings>;
|
||||
setSettings({ ...DEFAULT_SETTINGS, ...parsed });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[Settings] Failed to load settings:', error);
|
||||
// Continue with default settings
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
const updateSettings = useCallback(async (updates: Partial<Settings>) => {
|
||||
try {
|
||||
const newSettings = { ...settings, ...updates };
|
||||
setSettings(newSettings);
|
||||
await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(newSettings));
|
||||
} catch (error) {
|
||||
console.error('[Settings] Failed to save settings:', error);
|
||||
throw error;
|
||||
}
|
||||
}, [settings]);
|
||||
|
||||
const resetSettings = useCallback(async () => {
|
||||
try {
|
||||
setSettings(DEFAULT_SETTINGS);
|
||||
await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(DEFAULT_SETTINGS));
|
||||
} catch (error) {
|
||||
console.error('[Settings] Failed to reset settings:', error);
|
||||
throw error;
|
||||
}
|
||||
}, []);
|
||||
|
||||
return {
|
||||
settings,
|
||||
isLoading,
|
||||
updateSettings,
|
||||
resetSettings,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Learn more about light and dark modes:
|
||||
* https://docs.expo.dev/guides/color-schemes/
|
||||
*/
|
||||
|
||||
import { Colors } from '@/constants/theme';
|
||||
import { useColorScheme } from '@/hooks/use-color-scheme';
|
||||
|
||||
export function useThemeColor(
|
||||
props: { light?: string; dark?: string },
|
||||
colorName: keyof typeof Colors.light & keyof typeof Colors.dark
|
||||
) {
|
||||
const theme = useColorScheme() ?? 'light';
|
||||
const colorFromProps = props[theme];
|
||||
|
||||
if (colorFromProps) {
|
||||
return colorFromProps;
|
||||
} else {
|
||||
return Colors[theme][colorName];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import { useEffect, useRef, useState, useCallback } from 'react';
|
||||
import type {
|
||||
WSInboundMessage,
|
||||
WSOutboundMessage,
|
||||
SessionOutboundMessage
|
||||
} from '@server/server/messages';
|
||||
|
||||
export interface UseWebSocketReturn {
|
||||
isConnected: boolean;
|
||||
conversationId: string | null;
|
||||
send: (message: WSInboundMessage) => void;
|
||||
on: (type: SessionOutboundMessage['type'], handler: (message: SessionOutboundMessage) => void) => () => void;
|
||||
sendPing: () => void;
|
||||
sendUserMessage: (message: string) => void;
|
||||
}
|
||||
|
||||
export function useWebSocket(url: string, conversationId?: string | null): UseWebSocketReturn {
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const [currentConversationId, setCurrentConversationId] = useState<string | null>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const handlersRef = useRef<Map<SessionOutboundMessage['type'], Set<(message: SessionOutboundMessage) => void>>>(new Map());
|
||||
const reconnectTimeoutRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
|
||||
|
||||
const connect = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Add conversation ID to URL if provided
|
||||
const wsUrl = conversationId ? `${url}?conversationId=${conversationId}` : url;
|
||||
const ws = new WebSocket(wsUrl);
|
||||
|
||||
ws.onopen = () => {
|
||||
console.log('[WS] Connected to server');
|
||||
setIsConnected(true);
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
console.log('[WS] Disconnected from server');
|
||||
setIsConnected(false);
|
||||
|
||||
// Attempt to reconnect after 3 seconds
|
||||
reconnectTimeoutRef.current = setTimeout(() => {
|
||||
console.log('[WS] Attempting to reconnect...');
|
||||
connect();
|
||||
}, 3000);
|
||||
};
|
||||
|
||||
ws.onerror = (error) => {
|
||||
console.error('[WS] Error:', error);
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
try {
|
||||
const wsMessage: WSOutboundMessage = JSON.parse(event.data);
|
||||
|
||||
// Only session messages trigger handlers
|
||||
if (wsMessage.type === 'session') {
|
||||
const sessionMessage = wsMessage.message;
|
||||
console.log(`[WS] Received session message type: ${sessionMessage.type}`);
|
||||
|
||||
// Track conversation ID when loaded
|
||||
if (sessionMessage.type === 'conversation_loaded') {
|
||||
setCurrentConversationId(sessionMessage.payload.conversationId);
|
||||
}
|
||||
|
||||
// Call all registered handlers for this message type
|
||||
const handlers = handlersRef.current.get(sessionMessage.type);
|
||||
if (handlers) {
|
||||
handlers.forEach((handler) => {
|
||||
try {
|
||||
handler(sessionMessage);
|
||||
} catch (err) {
|
||||
console.error(`[WS] Error in handler for ${sessionMessage.type}:`, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// pong - just log
|
||||
console.log(`[WS] Received ${wsMessage.type}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[WS] Failed to parse message:', err);
|
||||
}
|
||||
};
|
||||
|
||||
wsRef.current = ws;
|
||||
} catch (err) {
|
||||
console.error('[WS] Failed to create WebSocket:', err);
|
||||
}
|
||||
}, [url, conversationId]);
|
||||
|
||||
useEffect(() => {
|
||||
connect();
|
||||
|
||||
return () => {
|
||||
if (reconnectTimeoutRef.current) {
|
||||
clearTimeout(reconnectTimeoutRef.current);
|
||||
}
|
||||
if (wsRef.current) {
|
||||
wsRef.current.close();
|
||||
}
|
||||
};
|
||||
}, [connect]);
|
||||
|
||||
const send = useCallback((message: WSInboundMessage) => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify(message));
|
||||
} else {
|
||||
console.warn('[WS] Cannot send message - not connected');
|
||||
}
|
||||
}, []);
|
||||
|
||||
const on = useCallback((
|
||||
type: SessionOutboundMessage['type'],
|
||||
handler: (message: SessionOutboundMessage) => void
|
||||
) => {
|
||||
if (!handlersRef.current.has(type)) {
|
||||
handlersRef.current.set(type, new Set());
|
||||
}
|
||||
handlersRef.current.get(type)!.add(handler);
|
||||
|
||||
// Return cleanup function
|
||||
return () => {
|
||||
const handlers = handlersRef.current.get(type);
|
||||
if (handlers) {
|
||||
handlers.delete(handler);
|
||||
if (handlers.size === 0) {
|
||||
handlersRef.current.delete(type);
|
||||
}
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const sendPing = useCallback(() => {
|
||||
send({ type: 'ping' });
|
||||
}, [send]);
|
||||
|
||||
const sendUserMessage = useCallback(
|
||||
(message: string) => {
|
||||
send({
|
||||
type: 'session',
|
||||
message: {
|
||||
type: 'user_text',
|
||||
text: message,
|
||||
},
|
||||
});
|
||||
},
|
||||
[send]
|
||||
);
|
||||
|
||||
return {
|
||||
isConnected,
|
||||
conversationId: currentConversationId,
|
||||
send,
|
||||
on,
|
||||
sendPing,
|
||||
sendUserMessage,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user