feat: refactor app structure and components for improved organization and functionality
- Moved components and hooks into a new directory structure under `src/app` for better organization. - Updated the main entry point to use `index.ts` and adjusted routing accordingly. - Introduced new components such as `ActiveProcesses`, `AgentStreamView`, and `ConnectionStatus` to enhance the app's functionality. - Removed deprecated components and hooks to streamline the codebase. - Updated styles to align with the new theme structure and ensure consistency across the app. This refactor aims to improve maintainability and scalability of the application.
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,183 @@
|
||||
import { useState, useRef, useEffect } from 'react';
|
||||
import { initialize, playPCMData } from '@speechmatics/expo-two-way-audio';
|
||||
|
||||
interface QueuedAudio {
|
||||
audioData: Blob;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resample PCM16 audio from 24kHz to 16kHz
|
||||
* OpenAI returns 24kHz, Speechmatics expects 16kHz
|
||||
*/
|
||||
function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array {
|
||||
// PCM16 = 2 bytes per sample
|
||||
const samples24k = pcm24k.length / 2;
|
||||
const samples16k = Math.floor(samples24k * 16000 / 24000);
|
||||
|
||||
const pcm16k = new Uint8Array(samples16k * 2);
|
||||
const ratio = 24000 / 16000; // 1.5
|
||||
|
||||
for (let i = 0; i < samples16k; i++) {
|
||||
const srcIndex = Math.floor(i * ratio) * 2;
|
||||
if (srcIndex + 1 < pcm24k.length) {
|
||||
pcm16k[i * 2] = pcm24k[srcIndex];
|
||||
pcm16k[i * 2 + 1] = pcm24k[srcIndex + 1];
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[AudioPlayer] Resampled PCM:', {
|
||||
input24k: pcm24k.length,
|
||||
output16k: pcm16k.length,
|
||||
durationMs: (samples16k / 16),
|
||||
});
|
||||
|
||||
return pcm16k;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio playback using Speechmatics two-way audio with echo cancellation
|
||||
*/
|
||||
export function useAudioPlayer() {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
const queueRef = useRef<QueuedAudio[]>([]);
|
||||
const isProcessingQueueRef = useRef(false);
|
||||
|
||||
// Initialize Speechmatics audio on mount
|
||||
useEffect(() => {
|
||||
const initializeAudio = async () => {
|
||||
try {
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)');
|
||||
} catch (error) {
|
||||
console.error('[AudioPlayer] Failed to initialize audio:', error);
|
||||
}
|
||||
};
|
||||
initializeAudio();
|
||||
}, []);
|
||||
|
||||
async function play(audioData: Blob): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Add to queue with its promise handlers
|
||||
queueRef.current.push({ audioData, resolve, reject });
|
||||
|
||||
// Start processing queue if not already processing
|
||||
if (!isProcessingQueueRef.current) {
|
||||
processQueue();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = true;
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
async function processNextInQueue(): Promise<void> {
|
||||
if (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
} else {
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audioData: Blob): Promise<number> {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
console.log(
|
||||
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
|
||||
);
|
||||
|
||||
if (!audioInitialized) {
|
||||
throw new Error('Audio not initialized');
|
||||
}
|
||||
|
||||
// Get PCM data from blob (server now sends PCM format)
|
||||
const arrayBuffer = await audioData.arrayBuffer();
|
||||
let pcm24k = new Uint8Array(arrayBuffer);
|
||||
|
||||
// Resample from 24kHz (OpenAI) to 16kHz (Speechmatics)
|
||||
const pcm16k = resamplePcm24kTo16k(pcm24k);
|
||||
|
||||
// Calculate total duration
|
||||
const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample
|
||||
const durationSec = samples / 16000; // 16kHz sample rate
|
||||
|
||||
const audioSizeKb = (pcm16k.length / 1024).toFixed(2);
|
||||
console.log('[AudioPlayer] 🔊 Playing audio:', audioSizeKb, 'KB, duration:', durationSec.toFixed(2), 's');
|
||||
|
||||
setIsPlaying(true);
|
||||
|
||||
// Play entire PCM data at once through Speechmatics
|
||||
playPCMData(pcm16k);
|
||||
|
||||
// Wait for playback to finish (estimate based on duration)
|
||||
setTimeout(() => {
|
||||
console.log('[AudioPlayer] ✅ Playback finished');
|
||||
setIsPlaying(false);
|
||||
resolve(durationSec);
|
||||
}, durationSec * 1000);
|
||||
|
||||
} catch (error) {
|
||||
console.error('[AudioPlayer] Error playing audio:', error);
|
||||
setIsPlaying(false);
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (isPlaying) {
|
||||
console.log('[AudioPlayer] 🛑 Stopping playback (interrupted)');
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
|
||||
// Reject all pending promises in the queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error('Playback stopped'));
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
function clearQueue(): void {
|
||||
// Reject all pending promises in the queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error('Queue cleared'));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
play,
|
||||
stop,
|
||||
isPlaying: () => isPlaying,
|
||||
clearQueue,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
import { useAudioRecorder as useExpoAudioRecorder, useAudioRecorderState, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
|
||||
import { Paths, File, Directory, FileInfo } from 'expo-file-system';
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
export interface AudioCaptureConfig {
|
||||
sampleRate?: number;
|
||||
numberOfChannels?: number;
|
||||
bitRate?: number;
|
||||
onAudioLevel?: (level: number) => void;
|
||||
onSpeechSegment?: (audioBlob: Blob) => void;
|
||||
enableContinuousRecording?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Workaround for Expo SDK 54 Android bug where audioRecorder.uri returns empty/zero-byte file
|
||||
* https://github.com/expo/expo/issues/39646
|
||||
*/
|
||||
async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
|
||||
try {
|
||||
console.log('[AudioRecorder] Searching for recording file created at:', createdAt.toISOString());
|
||||
|
||||
const audioDir = new Directory(Paths.cache, 'Audio');
|
||||
console.log('[AudioRecorder] Audio cache directory URI:', audioDir.uri);
|
||||
console.log('[AudioRecorder] Directory exists:', audioDir.exists);
|
||||
|
||||
if (!audioDir.exists) {
|
||||
console.log('[AudioRecorder] Audio cache directory does not exist');
|
||||
return null;
|
||||
}
|
||||
|
||||
const files = audioDir.list();
|
||||
console.log('[AudioRecorder] Found files in Audio cache:', files.length);
|
||||
|
||||
if (!files.length) {
|
||||
console.log('[AudioRecorder] No files found in Audio cache directory');
|
||||
return null;
|
||||
}
|
||||
|
||||
const validFiles = files
|
||||
.map(file => {
|
||||
const info = file.info();
|
||||
console.log('[AudioRecorder] File info:', {
|
||||
uri: info.uri,
|
||||
size: info.size,
|
||||
creationTime: info.creationTime ? new Date(info.creationTime).toISOString() : null,
|
||||
});
|
||||
return info;
|
||||
})
|
||||
.filter(f => f.size && f.size > 0);
|
||||
|
||||
console.log('[AudioRecorder] Valid files (size > 0):', validFiles.length);
|
||||
|
||||
if (validFiles.length === 0) {
|
||||
console.log('[AudioRecorder] No valid files found (all are zero-byte)');
|
||||
return null;
|
||||
}
|
||||
|
||||
let closest: FileInfo | null = null;
|
||||
let minDiff = Infinity;
|
||||
|
||||
for (const file of validFiles) {
|
||||
if (!file.creationTime || !file.uri) continue;
|
||||
const diff = Math.abs(file.creationTime - createdAt.getTime());
|
||||
console.log('[AudioRecorder] Time diff for file:', {
|
||||
uri: file.uri,
|
||||
diffMs: diff,
|
||||
});
|
||||
if (diff < minDiff) {
|
||||
closest = file;
|
||||
minDiff = diff;
|
||||
}
|
||||
}
|
||||
|
||||
if (closest) {
|
||||
const resultUri = closest.uri?.slice(0, -1) ?? null;
|
||||
console.log('[AudioRecorder] Found closest file:', {
|
||||
uri: resultUri,
|
||||
size: closest.size,
|
||||
timeDiffMs: minDiff,
|
||||
});
|
||||
return resultUri;
|
||||
}
|
||||
|
||||
console.log('[AudioRecorder] No closest file found');
|
||||
return null;
|
||||
} catch (e) {
|
||||
console.error('[AudioRecorder] Error finding actual recording file:', e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert audio file URI to Blob for WebSocket transmission
|
||||
* Returns a Blob-like object that works in React Native
|
||||
*/
|
||||
async function uriToBlob(uri: string): Promise<Blob> {
|
||||
// Use expo-file-system to read the file
|
||||
const file = new File(uri);
|
||||
const base64 = await file.base64();
|
||||
const size = file.size;
|
||||
|
||||
// React Native doesn't support creating Blobs from binary data
|
||||
// Create a Blob-like object that has the methods we need
|
||||
const blobLike = {
|
||||
type: 'audio/m4a',
|
||||
size: size,
|
||||
arrayBuffer: async () => {
|
||||
// Convert base64 to ArrayBuffer
|
||||
const binaryString = atob(base64);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
return bytes.buffer;
|
||||
},
|
||||
} as Blob;
|
||||
|
||||
return blobLike;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio recording with configuration matching web version
|
||||
* Matches the web app's audio constraints:
|
||||
* - 16000 sample rate (optimal for speech/Whisper)
|
||||
* - 1 channel (mono)
|
||||
* - Echo cancellation, noise suppression, auto gain control (voice_communication on Android)
|
||||
*/
|
||||
export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [recordingStartTime, setRecordingStartTime] = useState<Date | null>(null);
|
||||
|
||||
// Create recording options matching web app constraints
|
||||
const recordingOptions: RecordingOptions = {
|
||||
...RecordingPresets.HIGH_QUALITY,
|
||||
sampleRate: config?.sampleRate || 16000,
|
||||
numberOfChannels: config?.numberOfChannels || 1,
|
||||
bitRate: config?.bitRate || 128000,
|
||||
extension: '.m4a',
|
||||
isMeteringEnabled: !!config?.onAudioLevel, // Enable metering if callback provided
|
||||
android: {
|
||||
extension: '.m4a',
|
||||
outputFormat: 'mpeg4',
|
||||
audioEncoder: 'aac',
|
||||
sampleRate: config?.sampleRate || 16000,
|
||||
audioSource: 'voice_communication', // Enables echo cancellation, noise suppression, auto gain control
|
||||
},
|
||||
ios: {
|
||||
extension: '.m4a',
|
||||
audioQuality: 127, // High quality
|
||||
sampleRate: config?.sampleRate || 16000,
|
||||
linearPCMBitDepth: 16,
|
||||
linearPCMIsBigEndian: false,
|
||||
linearPCMIsFloat: false,
|
||||
},
|
||||
web: {
|
||||
mimeType: 'audio/webm;codecs=opus',
|
||||
bitsPerSecond: config?.bitRate || 128000,
|
||||
},
|
||||
};
|
||||
|
||||
const audioRecorder = useExpoAudioRecorder(recordingOptions);
|
||||
const recorderState = useAudioRecorderState(audioRecorder, 100);
|
||||
|
||||
// Monitor audio levels if metering is enabled
|
||||
useEffect(() => {
|
||||
if (!config?.onAudioLevel || !isRecording) return;
|
||||
|
||||
const interval = setInterval(() => {
|
||||
const metering = recorderState.metering;
|
||||
if (metering !== undefined && metering !== null) {
|
||||
// Normalize metering value (typically ranges from -160 to 0 dB)
|
||||
// Convert to 0-1 range where 0 is silence and 1 is loud
|
||||
// We'll use -40 dB as the threshold for "loud"
|
||||
const normalized = Math.max(0, Math.min(1, (metering + 40) / 40));
|
||||
config.onAudioLevel?.(normalized);
|
||||
}
|
||||
}, 100); // Check every 100ms
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [isRecording, config, recorderState.metering]);
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (isRecording) {
|
||||
throw new Error('Already recording');
|
||||
}
|
||||
|
||||
try {
|
||||
// Request microphone permissions
|
||||
console.log('[AudioRecorder] Requesting recording permissions...');
|
||||
const permissionResponse = await requestRecordingPermissionsAsync();
|
||||
|
||||
if (!permissionResponse.granted) {
|
||||
throw new Error('Microphone permission denied. Please enable microphone access in your device settings.');
|
||||
}
|
||||
|
||||
// Configure audio mode for recording
|
||||
console.log('[AudioRecorder] Configuring audio mode...');
|
||||
await setAudioModeAsync({
|
||||
playsInSilentMode: true,
|
||||
allowsRecording: true,
|
||||
});
|
||||
|
||||
console.log('[AudioRecorder] Starting recording with options:', {
|
||||
sampleRate: recordingOptions.sampleRate,
|
||||
numberOfChannels: recordingOptions.numberOfChannels,
|
||||
bitRate: recordingOptions.bitRate,
|
||||
});
|
||||
|
||||
const startTime = new Date();
|
||||
setRecordingStartTime(startTime);
|
||||
|
||||
// Prepare the recorder before recording (required step)
|
||||
console.log('[AudioRecorder] Preparing recorder...');
|
||||
await audioRecorder.prepareToRecordAsync();
|
||||
|
||||
console.log('[AudioRecorder] Starting recording...');
|
||||
await audioRecorder.record();
|
||||
setIsRecording(true);
|
||||
|
||||
console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
|
||||
console.log('[AudioRecorder] Recorder isRecording:', audioRecorder.isRecording);
|
||||
} catch (error: any) {
|
||||
console.error('[AudioRecorder] Failed to start recording:', error);
|
||||
throw new Error(`Failed to start audio recording: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<Blob> {
|
||||
if (!isRecording) {
|
||||
throw new Error('Not recording');
|
||||
}
|
||||
|
||||
try {
|
||||
// Stop recording
|
||||
await audioRecorder.stop();
|
||||
setIsRecording(false);
|
||||
|
||||
// Get URI from recorder
|
||||
let uri = audioRecorder.uri;
|
||||
console.log('[AudioRecorder] Initial URI from recorder:', uri);
|
||||
|
||||
// Workaround for Expo SDK 54 Android bug - find actual recording file
|
||||
if (recordingStartTime && (!uri || uri === '')) {
|
||||
console.log('[AudioRecorder] Using workaround to find actual recording file...');
|
||||
const actualUri = await getActualRecordingUri(recordingStartTime);
|
||||
if (actualUri) {
|
||||
uri = actualUri;
|
||||
console.log('[AudioRecorder] Found actual recording URI:', uri);
|
||||
}
|
||||
}
|
||||
|
||||
if (!uri || uri === '') {
|
||||
throw new Error('No recording URI found');
|
||||
}
|
||||
|
||||
// Get file info
|
||||
const file = new File(uri);
|
||||
const exists = file.exists;
|
||||
console.log('[AudioRecorder] File exists:', exists);
|
||||
|
||||
if (!exists) {
|
||||
throw new Error('Recording file does not exist');
|
||||
}
|
||||
|
||||
// Convert URI to Blob
|
||||
const audioBlob = await uriToBlob(uri);
|
||||
|
||||
console.log('[AudioRecorder] Recording converted to blob:', {
|
||||
size: audioBlob.size,
|
||||
type: audioBlob.type,
|
||||
});
|
||||
|
||||
// Clean up the temporary file
|
||||
file.delete();
|
||||
|
||||
// Reset start time
|
||||
setRecordingStartTime(null);
|
||||
|
||||
return audioBlob;
|
||||
} catch (error: any) {
|
||||
setIsRecording(false);
|
||||
setRecordingStartTime(null);
|
||||
console.error('[AudioRecorder] Failed to stop recording:', error);
|
||||
throw new Error(`Failed to stop audio recording: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function getSupportedMimeType(): string | null {
|
||||
// On native platforms, expo-audio uses m4a/AAC
|
||||
// On web, it can use webm/opus
|
||||
return 'audio/m4a';
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
isRecording: () => isRecording,
|
||||
getSupportedMimeType,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { useColorScheme } from 'react-native';
|
||||
@@ -0,0 +1,21 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useColorScheme as useRNColorScheme } from 'react-native';
|
||||
|
||||
/**
|
||||
* To support static rendering, this value needs to be re-calculated on the client side for web
|
||||
*/
|
||||
export function useColorScheme() {
|
||||
const [hasHydrated, setHasHydrated] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
setHasHydrated(true);
|
||||
}, []);
|
||||
|
||||
const colorScheme = useRNColorScheme();
|
||||
|
||||
if (hasHydrated) {
|
||||
return colorScheme;
|
||||
}
|
||||
|
||||
return 'light';
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import AsyncStorage from '@react-native-async-storage/async-storage';
|
||||
|
||||
const STORAGE_KEY = '@voice-dev:settings';
|
||||
|
||||
export interface Settings {
|
||||
serverUrl: string;
|
||||
useSpeaker: boolean;
|
||||
keepScreenOn: boolean;
|
||||
theme: 'dark' | 'light' | 'auto';
|
||||
}
|
||||
|
||||
const DEFAULT_SETTINGS: Settings = {
|
||||
serverUrl: 'wss://mohameds-macbook-pro.tail8fe838.ts.net/ws',
|
||||
useSpeaker: true,
|
||||
keepScreenOn: true,
|
||||
theme: 'dark',
|
||||
};
|
||||
|
||||
export interface UseSettingsReturn {
|
||||
settings: Settings;
|
||||
isLoading: boolean;
|
||||
updateSettings: (updates: Partial<Settings>) => Promise<void>;
|
||||
resetSettings: () => Promise<void>;
|
||||
}
|
||||
|
||||
export function useSettings(): UseSettingsReturn {
|
||||
const [settings, setSettings] = useState<Settings>(DEFAULT_SETTINGS);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
|
||||
// Load settings from AsyncStorage on mount
|
||||
useEffect(() => {
|
||||
loadSettings();
|
||||
}, []);
|
||||
|
||||
async function loadSettings() {
|
||||
try {
|
||||
const stored = await AsyncStorage.getItem(STORAGE_KEY);
|
||||
if (stored) {
|
||||
const parsed = JSON.parse(stored) as Partial<Settings>;
|
||||
setSettings({ ...DEFAULT_SETTINGS, ...parsed });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[Settings] Failed to load settings:', error);
|
||||
// Continue with default settings
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
const updateSettings = useCallback(async (updates: Partial<Settings>) => {
|
||||
try {
|
||||
const newSettings = { ...settings, ...updates };
|
||||
setSettings(newSettings);
|
||||
await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(newSettings));
|
||||
} catch (error) {
|
||||
console.error('[Settings] Failed to save settings:', error);
|
||||
throw error;
|
||||
}
|
||||
}, [settings]);
|
||||
|
||||
const resetSettings = useCallback(async () => {
|
||||
try {
|
||||
setSettings(DEFAULT_SETTINGS);
|
||||
await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(DEFAULT_SETTINGS));
|
||||
} catch (error) {
|
||||
console.error('[Settings] Failed to reset settings:', error);
|
||||
throw error;
|
||||
}
|
||||
}, []);
|
||||
|
||||
return {
|
||||
settings,
|
||||
isLoading,
|
||||
updateSettings,
|
||||
resetSettings,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
import { useState, useEffect, useRef, useCallback } from "react";
|
||||
import {
|
||||
initialize,
|
||||
toggleRecording,
|
||||
tearDown,
|
||||
useExpoTwoWayAudioEventListener,
|
||||
type MicrophoneDataCallback,
|
||||
type VolumeLevelCallback,
|
||||
} from "@speechmatics/expo-two-way-audio";
|
||||
|
||||
export interface SpeechmaticsAudioConfig {
|
||||
onAudioSegment?: (audioData: string) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
volumeThreshold: number; // Volume threshold for speech detection (0-1)
|
||||
silenceDuration: number; // ms of silence before ending segment
|
||||
speechConfirmationDuration: number; // ms of sustained speech before confirming
|
||||
detectionGracePeriod: number; // ms grace period for volume dips during detection
|
||||
}
|
||||
|
||||
export interface SpeechmaticsAudio {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
isActive: boolean;
|
||||
isSpeaking: boolean;
|
||||
isDetecting: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert raw PCM data to WAV format by adding WAV headers
|
||||
*/
|
||||
function pcmToWav(
|
||||
pcmBase64: string,
|
||||
sampleRate: number = 16000,
|
||||
channels: number = 1,
|
||||
bitsPerSample: number = 16
|
||||
): string {
|
||||
const pcmBinary = atob(pcmBase64);
|
||||
const pcmLength = pcmBinary.length;
|
||||
|
||||
const byteRate = sampleRate * channels * (bitsPerSample / 8);
|
||||
const blockAlign = channels * (bitsPerSample / 8);
|
||||
const dataSize = pcmLength;
|
||||
const fileSize = 44 + dataSize;
|
||||
|
||||
const wavBuffer = new Uint8Array(fileSize);
|
||||
const view = new DataView(wavBuffer.buffer);
|
||||
|
||||
let offset = 0;
|
||||
|
||||
// RIFF chunk descriptor
|
||||
writeString(view, offset, "RIFF");
|
||||
offset += 4;
|
||||
view.setUint32(offset, fileSize - 8, true);
|
||||
offset += 4;
|
||||
writeString(view, offset, "WAVE");
|
||||
offset += 4;
|
||||
|
||||
// fmt sub-chunk
|
||||
writeString(view, offset, "fmt ");
|
||||
offset += 4;
|
||||
view.setUint32(offset, 16, true);
|
||||
offset += 4;
|
||||
view.setUint16(offset, 1, true);
|
||||
offset += 2;
|
||||
view.setUint16(offset, channels, true);
|
||||
offset += 2;
|
||||
view.setUint32(offset, sampleRate, true);
|
||||
offset += 4;
|
||||
view.setUint32(offset, byteRate, true);
|
||||
offset += 4;
|
||||
view.setUint16(offset, blockAlign, true);
|
||||
offset += 2;
|
||||
view.setUint16(offset, bitsPerSample, true);
|
||||
offset += 2;
|
||||
|
||||
// data sub-chunk
|
||||
writeString(view, offset, "data");
|
||||
offset += 4;
|
||||
view.setUint32(offset, dataSize, true);
|
||||
offset += 4;
|
||||
|
||||
// Copy PCM data
|
||||
for (let i = 0; i < pcmLength; i++) {
|
||||
wavBuffer[offset + i] = pcmBinary.charCodeAt(i);
|
||||
}
|
||||
|
||||
// Convert to base64
|
||||
let binary = "";
|
||||
for (let i = 0; i < wavBuffer.length; i++) {
|
||||
binary += String.fromCharCode(wavBuffer[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function writeString(view: DataView, offset: number, str: string): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
function uint8ArrayToBase64(bytes: Uint8Array): string {
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
|
||||
const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
|
||||
const result = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const arr of arrays) {
|
||||
result.set(arr, offset);
|
||||
offset += arr.length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio
|
||||
*/
|
||||
export function useSpeechmaticsAudio(
|
||||
config: SpeechmaticsAudioConfig
|
||||
): SpeechmaticsAudio {
|
||||
const [isActive, setIsActive] = useState(false);
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
const [isDetecting, setIsDetecting] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
|
||||
const audioBufferRef = useRef<Uint8Array[]>([]);
|
||||
const silenceStartRef = useRef<number | null>(null);
|
||||
const isSpeakingRef = useRef(false);
|
||||
const speechDetectionStartRef = useRef<number | null>(null);
|
||||
const speechConfirmedRef = useRef(false);
|
||||
const detectionSilenceStartRef = useRef<number | null>(null);
|
||||
|
||||
const VOLUME_THRESHOLD = config.volumeThreshold;
|
||||
const SILENCE_DURATION_MS = config.silenceDuration;
|
||||
const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration;
|
||||
const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod;
|
||||
|
||||
// Initialize audio on mount
|
||||
useEffect(() => {
|
||||
const initializeAudio = async () => {
|
||||
try {
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
console.log("[SpeechmaticsAudio] Audio initialized");
|
||||
} catch (error) {
|
||||
console.error("[SpeechmaticsAudio] Initialization error:", error);
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
}
|
||||
};
|
||||
initializeAudio();
|
||||
|
||||
return () => {
|
||||
tearDown();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Listen to microphone data
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
useCallback<MicrophoneDataCallback>(
|
||||
(event) => {
|
||||
if (!isActive) return;
|
||||
|
||||
const pcmData: Uint8Array = event.data;
|
||||
|
||||
// Buffer the audio chunk if we're detecting or speaking
|
||||
// Start buffering from first spike to capture beginning of speech
|
||||
if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
|
||||
audioBufferRef.current.push(pcmData);
|
||||
}
|
||||
},
|
||||
[isActive]
|
||||
)
|
||||
);
|
||||
|
||||
// Listen to volume level for VAD
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
useCallback<VolumeLevelCallback>(
|
||||
(event) => {
|
||||
if (!isActive) return;
|
||||
|
||||
const volume: number = event.data;
|
||||
const speechDetected = volume > VOLUME_THRESHOLD;
|
||||
|
||||
// console.log('[SpeechmaticsAudio] Volume:', volume.toFixed(6));
|
||||
|
||||
if (speechDetected && !isSpeakingRef.current && !speechConfirmedRef.current) {
|
||||
// Initial speech detection - start tracking
|
||||
if (speechDetectionStartRef.current === null) {
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech started (volume:",
|
||||
volume.toFixed(4),
|
||||
")"
|
||||
);
|
||||
speechDetectionStartRef.current = Date.now();
|
||||
detectionSilenceStartRef.current = null;
|
||||
audioBufferRef.current = [];
|
||||
setIsDetecting(true);
|
||||
} else {
|
||||
// Volume is back above threshold - reset grace period
|
||||
detectionSilenceStartRef.current = null;
|
||||
|
||||
// Check if speech has been sustained long enough
|
||||
const speechDuration = Date.now() - speechDetectionStartRef.current;
|
||||
if (speechDuration >= SPEECH_CONFIRMATION_MS) {
|
||||
// Speech CONFIRMED
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech confirmed after",
|
||||
speechDuration,
|
||||
"ms"
|
||||
);
|
||||
isSpeakingRef.current = true;
|
||||
speechConfirmedRef.current = true;
|
||||
silenceStartRef.current = null;
|
||||
setIsDetecting(false);
|
||||
setIsSpeaking(true);
|
||||
config.onSpeechStart?.();
|
||||
}
|
||||
}
|
||||
} else if (speechDetected && isSpeakingRef.current && speechConfirmedRef.current) {
|
||||
// Continuing confirmed speech
|
||||
silenceStartRef.current = null;
|
||||
} else if (!speechDetected && !speechConfirmedRef.current && speechDetectionStartRef.current !== null) {
|
||||
// Volume dropped during detection phase - apply grace period
|
||||
if (detectionSilenceStartRef.current === null) {
|
||||
detectionSilenceStartRef.current = Date.now();
|
||||
} else {
|
||||
const graceDuration = Date.now() - detectionSilenceStartRef.current;
|
||||
if (graceDuration >= DETECTION_GRACE_PERIOD_MS) {
|
||||
// Grace period expired - cancel detection
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech detection cancelled after",
|
||||
graceDuration,
|
||||
"ms grace period"
|
||||
);
|
||||
speechDetectionStartRef.current = null;
|
||||
detectionSilenceStartRef.current = null;
|
||||
audioBufferRef.current = [];
|
||||
setIsDetecting(false);
|
||||
}
|
||||
}
|
||||
} else if (!speechDetected && isSpeakingRef.current && speechConfirmedRef.current) {
|
||||
// Potential speech END
|
||||
if (silenceStartRef.current === null) {
|
||||
silenceStartRef.current = Date.now();
|
||||
} else {
|
||||
const silenceDuration = Date.now() - silenceStartRef.current;
|
||||
if (silenceDuration >= SILENCE_DURATION_MS) {
|
||||
// Speech END confirmed
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech ended after",
|
||||
silenceDuration,
|
||||
"ms silence"
|
||||
);
|
||||
isSpeakingRef.current = false;
|
||||
speechConfirmedRef.current = false;
|
||||
speechDetectionStartRef.current = null;
|
||||
silenceStartRef.current = null;
|
||||
setIsSpeaking(false);
|
||||
config.onSpeechEnd?.();
|
||||
|
||||
// Send buffered audio segment
|
||||
if (audioBufferRef.current.length > 0) {
|
||||
const combinedBinary = concatenateUint8Arrays(
|
||||
audioBufferRef.current
|
||||
);
|
||||
const combinedPcmBase64 = uint8ArrayToBase64(combinedBinary);
|
||||
const wavAudio = pcmToWav(combinedPcmBase64, 16000, 1, 16);
|
||||
config.onAudioSegment?.(wavAudio);
|
||||
audioBufferRef.current = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
[isActive, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config]
|
||||
)
|
||||
);
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (!audioInitialized) {
|
||||
throw new Error("Audio not initialized");
|
||||
}
|
||||
|
||||
if (isActive) {
|
||||
console.log("[SpeechmaticsAudio] Already active");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
console.log("[SpeechmaticsAudio] Starting audio capture...");
|
||||
|
||||
// Start recording
|
||||
toggleRecording(true);
|
||||
|
||||
setIsActive(true);
|
||||
console.log("[SpeechmaticsAudio] Audio capture started successfully");
|
||||
} catch (error) {
|
||||
console.error("[SpeechmaticsAudio] Start error:", error);
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
await stop();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<void> {
|
||||
console.log("[SpeechmaticsAudio] Stopping audio capture...");
|
||||
|
||||
// Stop recording
|
||||
toggleRecording(false);
|
||||
|
||||
// Reset state
|
||||
audioBufferRef.current = [];
|
||||
isSpeakingRef.current = false;
|
||||
speechConfirmedRef.current = false;
|
||||
speechDetectionStartRef.current = null;
|
||||
detectionSilenceStartRef.current = null;
|
||||
silenceStartRef.current = null;
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
|
||||
console.log("[SpeechmaticsAudio] Audio capture stopped");
|
||||
}
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (isActive) {
|
||||
stop();
|
||||
}
|
||||
};
|
||||
}, [isActive]);
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
isActive,
|
||||
isSpeaking,
|
||||
isDetecting,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Learn more about light and dark modes:
|
||||
* https://docs.expo.dev/guides/color-schemes/
|
||||
*/
|
||||
|
||||
import { Colors } from '@/constants/theme';
|
||||
import { useColorScheme } from '@/hooks/use-color-scheme';
|
||||
|
||||
export function useThemeColor(
|
||||
props: { light?: string; dark?: string },
|
||||
colorName: keyof typeof Colors.light & keyof typeof Colors.dark
|
||||
) {
|
||||
const theme = useColorScheme() ?? 'light';
|
||||
const colorFromProps = props[theme];
|
||||
|
||||
if (colorFromProps) {
|
||||
return colorFromProps;
|
||||
} else {
|
||||
return Colors[theme][colorName];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import { useEffect, useRef, useState, useCallback } from 'react';
|
||||
import type {
|
||||
WSInboundMessage,
|
||||
WSOutboundMessage,
|
||||
SessionOutboundMessage
|
||||
} from '@server/server/messages';
|
||||
|
||||
export interface UseWebSocketReturn {
|
||||
isConnected: boolean;
|
||||
conversationId: string | null;
|
||||
send: (message: WSInboundMessage) => void;
|
||||
on: (type: SessionOutboundMessage['type'], handler: (message: SessionOutboundMessage) => void) => () => void;
|
||||
sendPing: () => void;
|
||||
sendUserMessage: (message: string) => void;
|
||||
}
|
||||
|
||||
export function useWebSocket(url: string, conversationId?: string | null): UseWebSocketReturn {
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const [currentConversationId, setCurrentConversationId] = useState<string | null>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const handlersRef = useRef<Map<SessionOutboundMessage['type'], Set<(message: SessionOutboundMessage) => void>>>(new Map());
|
||||
const reconnectTimeoutRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
|
||||
|
||||
const connect = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Add conversation ID to URL if provided
|
||||
const wsUrl = conversationId ? `${url}?conversationId=${conversationId}` : url;
|
||||
const ws = new WebSocket(wsUrl);
|
||||
|
||||
ws.onopen = () => {
|
||||
console.log('[WS] Connected to server');
|
||||
setIsConnected(true);
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
console.log('[WS] Disconnected from server');
|
||||
setIsConnected(false);
|
||||
|
||||
// Attempt to reconnect after 3 seconds
|
||||
reconnectTimeoutRef.current = setTimeout(() => {
|
||||
console.log('[WS] Attempting to reconnect...');
|
||||
connect();
|
||||
}, 3000);
|
||||
};
|
||||
|
||||
ws.onerror = (error) => {
|
||||
console.error('[WS] Error:', error);
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
try {
|
||||
const wsMessage: WSOutboundMessage = JSON.parse(event.data);
|
||||
|
||||
// Only session messages trigger handlers
|
||||
if (wsMessage.type === 'session') {
|
||||
const sessionMessage = wsMessage.message;
|
||||
// console.log(`[WS] Received session message type: ${sessionMessage.type}`);
|
||||
|
||||
// Track conversation ID when loaded
|
||||
if (sessionMessage.type === 'conversation_loaded') {
|
||||
setCurrentConversationId(sessionMessage.payload.conversationId);
|
||||
}
|
||||
|
||||
// Call all registered handlers for this message type
|
||||
const handlers = handlersRef.current.get(sessionMessage.type);
|
||||
if (handlers) {
|
||||
handlers.forEach((handler) => {
|
||||
try {
|
||||
handler(sessionMessage);
|
||||
} catch (err) {
|
||||
console.error(`[WS] Error in handler for ${sessionMessage.type}:`, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// pong - just log
|
||||
console.log(`[WS] Received ${wsMessage.type}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[WS] Failed to parse message:', err);
|
||||
}
|
||||
};
|
||||
|
||||
wsRef.current = ws;
|
||||
} catch (err) {
|
||||
console.error('[WS] Failed to create WebSocket:', err);
|
||||
}
|
||||
}, [url, conversationId]);
|
||||
|
||||
useEffect(() => {
|
||||
connect();
|
||||
|
||||
return () => {
|
||||
if (reconnectTimeoutRef.current) {
|
||||
clearTimeout(reconnectTimeoutRef.current);
|
||||
}
|
||||
if (wsRef.current) {
|
||||
wsRef.current.close();
|
||||
}
|
||||
};
|
||||
}, [connect]);
|
||||
|
||||
const send = useCallback((message: WSInboundMessage) => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify(message));
|
||||
} else {
|
||||
console.warn('[WS] Cannot send message - not connected');
|
||||
}
|
||||
}, []);
|
||||
|
||||
const on = useCallback((
|
||||
type: SessionOutboundMessage['type'],
|
||||
handler: (message: SessionOutboundMessage) => void
|
||||
) => {
|
||||
if (!handlersRef.current.has(type)) {
|
||||
handlersRef.current.set(type, new Set());
|
||||
}
|
||||
handlersRef.current.get(type)!.add(handler);
|
||||
|
||||
// Return cleanup function
|
||||
return () => {
|
||||
const handlers = handlersRef.current.get(type);
|
||||
if (handlers) {
|
||||
handlers.delete(handler);
|
||||
if (handlers.size === 0) {
|
||||
handlersRef.current.delete(type);
|
||||
}
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const sendPing = useCallback(() => {
|
||||
send({ type: 'ping' });
|
||||
}, [send]);
|
||||
|
||||
const sendUserMessage = useCallback(
|
||||
(message: string) => {
|
||||
send({
|
||||
type: 'session',
|
||||
message: {
|
||||
type: 'user_text',
|
||||
text: message,
|
||||
},
|
||||
});
|
||||
},
|
||||
[send]
|
||||
);
|
||||
|
||||
return {
|
||||
isConnected,
|
||||
conversationId: currentConversationId,
|
||||
send,
|
||||
on,
|
||||
sendPing,
|
||||
sendUserMessage,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user