feat: migrate from NativeWind to react-native-unistyles and add Speechmatics audio
🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Mohamed Boudra
@@ -1,7 +1,5 @@
|
||||
import { useAudioPlayer as useExpoAudioPlayer, setAudioModeAsync } from 'expo-audio';
|
||||
import { Paths, File } from 'expo-file-system';
|
||||
import { useState, useRef, useEffect } from 'react';
|
||||
import { Platform } from 'react-native';
|
||||
import { initialize, playPCMData } from '@speechmatics/expo-two-way-audio';
|
||||
|
||||
interface QueuedAudio {
|
||||
audioData: Blob;
|
||||
@@ -9,75 +7,57 @@ interface QueuedAudio {
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
interface AudioPlayerOptions {
|
||||
useSpeaker?: boolean;
|
||||
/**
|
||||
* Resample PCM16 audio from 24kHz to 16kHz
|
||||
* OpenAI returns 24kHz, Speechmatics expects 16kHz
|
||||
*/
|
||||
function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array {
|
||||
// PCM16 = 2 bytes per sample
|
||||
const samples24k = pcm24k.length / 2;
|
||||
const samples16k = Math.floor(samples24k * 16000 / 24000);
|
||||
|
||||
const pcm16k = new Uint8Array(samples16k * 2);
|
||||
const ratio = 24000 / 16000; // 1.5
|
||||
|
||||
for (let i = 0; i < samples16k; i++) {
|
||||
const srcIndex = Math.floor(i * ratio) * 2;
|
||||
if (srcIndex + 1 < pcm24k.length) {
|
||||
pcm16k[i * 2] = pcm24k[srcIndex];
|
||||
pcm16k[i * 2 + 1] = pcm24k[srcIndex + 1];
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[AudioPlayer] Resampled PCM:', {
|
||||
input24k: pcm24k.length,
|
||||
output16k: pcm16k.length,
|
||||
durationMs: (samples16k / 16),
|
||||
});
|
||||
|
||||
return pcm16k;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Blob to file URI for expo-audio playback
|
||||
* Hook for audio playback using Speechmatics two-way audio with echo cancellation
|
||||
*/
|
||||
async function blobToFile(blob: Blob): Promise<File> {
|
||||
// Read blob as array buffer
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
|
||||
// Create temporary file in cache directory
|
||||
const fileName = `audio_${Date.now()}.mp3`;
|
||||
const file = new File(Paths.cache, fileName);
|
||||
|
||||
// Write array buffer to file
|
||||
const uint8Array = new Uint8Array(arrayBuffer);
|
||||
file.create();
|
||||
const stream = file.writableStream();
|
||||
const writer = stream.getWriter();
|
||||
await writer.write(uint8Array);
|
||||
await writer.close();
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio playback with queue system matching web version functionality
|
||||
*/
|
||||
export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
const player = useExpoAudioPlayer();
|
||||
export function useAudioPlayer() {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [isPaused, setIsPaused] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
const queueRef = useRef<QueuedAudio[]>([]);
|
||||
const isProcessingQueueRef = useRef(false);
|
||||
const currentRejectRef = useRef<((error: Error) => void) | null>(null);
|
||||
const currentFileRef = useRef<File | null>(null);
|
||||
|
||||
// Configure audio mode based on speaker/earpiece preference
|
||||
// Initialize Speechmatics audio on mount
|
||||
useEffect(() => {
|
||||
async function configureAudioMode() {
|
||||
const initializeAudio = async () => {
|
||||
try {
|
||||
if (Platform.OS === 'android') {
|
||||
// On Android, use shouldRouteThroughEarpiece
|
||||
await setAudioModeAsync({
|
||||
shouldRouteThroughEarpiece: !options?.useSpeaker,
|
||||
});
|
||||
} else if (Platform.OS === 'ios') {
|
||||
// On iOS, the audio routing is controlled by the AVAudioSession category
|
||||
// The default category should work, but we can configure it if needed
|
||||
// For now, we rely on the iOS default behavior which routes to the receiver
|
||||
// unless the user has changed the route (e.g., via speaker button in Control Center)
|
||||
}
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)');
|
||||
} catch (error) {
|
||||
console.warn('[AudioPlayer] Failed to configure audio mode:', error);
|
||||
console.error('[AudioPlayer] Failed to initialize audio:', error);
|
||||
}
|
||||
}
|
||||
|
||||
configureAudioMode();
|
||||
}, [options?.useSpeaker]);
|
||||
|
||||
// Monitor playback status
|
||||
useEffect(() => {
|
||||
if (!player.playing && isPlaying && !isPaused) {
|
||||
// Playback finished
|
||||
setIsPlaying(false);
|
||||
processNextInQueue();
|
||||
}
|
||||
}, [player.playing, isPlaying, isPaused]);
|
||||
};
|
||||
initializeAudio();
|
||||
}, []);
|
||||
|
||||
async function play(audioData: Blob): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -127,92 +107,55 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
|
||||
async function playAudio(audioData: Blob): Promise<number> {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
currentRejectRef.current = reject;
|
||||
|
||||
try {
|
||||
console.log(
|
||||
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
|
||||
);
|
||||
|
||||
// Convert blob to file
|
||||
const file = await blobToFile(audioData);
|
||||
currentFileRef.current = file;
|
||||
if (!audioInitialized) {
|
||||
throw new Error('Audio not initialized');
|
||||
}
|
||||
|
||||
// Get PCM data from blob (server now sends PCM format)
|
||||
const arrayBuffer = await audioData.arrayBuffer();
|
||||
let pcm24k = new Uint8Array(arrayBuffer);
|
||||
|
||||
// Resample from 24kHz (OpenAI) to 16kHz (Speechmatics)
|
||||
const pcm16k = resamplePcm24kTo16k(pcm24k);
|
||||
|
||||
// Calculate total duration
|
||||
const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample
|
||||
const durationSec = samples / 16000; // 16kHz sample rate
|
||||
|
||||
const audioSizeKb = (pcm16k.length / 1024).toFixed(2);
|
||||
console.log('[AudioPlayer] 🔊 Playing audio:', audioSizeKb, 'KB, duration:', durationSec.toFixed(2), 's');
|
||||
|
||||
setIsPlaying(true);
|
||||
setIsPaused(false);
|
||||
|
||||
// Load and play audio
|
||||
await player.replace({ uri: file.uri });
|
||||
player.play();
|
||||
// Play entire PCM data at once through Speechmatics
|
||||
playPCMData(pcm16k);
|
||||
|
||||
// Wait for playback to finish
|
||||
const checkInterval = setInterval(() => {
|
||||
if (!player.playing && player.duration > 0) {
|
||||
clearInterval(checkInterval);
|
||||
|
||||
const duration = player.duration;
|
||||
console.log(`[AudioPlayer] Playback finished (duration: ${duration}s)`);
|
||||
|
||||
setIsPlaying(false);
|
||||
currentRejectRef.current = null;
|
||||
|
||||
// Clean up temporary file
|
||||
if (currentFileRef.current) {
|
||||
try {
|
||||
currentFileRef.current.delete();
|
||||
} catch (err: any) {
|
||||
console.warn('[AudioPlayer] Failed to delete temp file:', err);
|
||||
}
|
||||
currentFileRef.current = null;
|
||||
}
|
||||
|
||||
resolve(duration);
|
||||
}
|
||||
}, 100);
|
||||
// Wait for playback to finish (estimate based on duration)
|
||||
setTimeout(() => {
|
||||
console.log('[AudioPlayer] ✅ Playback finished');
|
||||
setIsPlaying(false);
|
||||
resolve(durationSec);
|
||||
}, durationSec * 1000);
|
||||
|
||||
} catch (error) {
|
||||
console.error('[AudioPlayer] Error playing audio:', error);
|
||||
setIsPlaying(false);
|
||||
currentRejectRef.current = null;
|
||||
|
||||
// Clean up temporary file
|
||||
if (currentFileRef.current) {
|
||||
try {
|
||||
currentFileRef.current.delete();
|
||||
} catch (err: any) {
|
||||
console.warn('[AudioPlayer] Failed to delete temp file:', err);
|
||||
}
|
||||
currentFileRef.current = null;
|
||||
}
|
||||
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
// Stop currently playing audio
|
||||
if (player) {
|
||||
player.pause();
|
||||
if (isPlaying) {
|
||||
console.log('[AudioPlayer] 🛑 Stopping playback (interrupted)');
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
setIsPaused(false);
|
||||
|
||||
// Clean up temporary file
|
||||
if (currentFileRef.current) {
|
||||
try {
|
||||
currentFileRef.current.delete();
|
||||
} catch (err: any) {
|
||||
console.warn('[AudioPlayer] Failed to delete temp file:', err);
|
||||
}
|
||||
currentFileRef.current = null;
|
||||
}
|
||||
|
||||
// Reject the current playing promise if it exists
|
||||
if (currentRejectRef.current) {
|
||||
currentRejectRef.current(new Error('Playback stopped'));
|
||||
currentRejectRef.current = null;
|
||||
}
|
||||
|
||||
// Reject all pending promises in the queue
|
||||
while (queueRef.current.length > 0) {
|
||||
@@ -223,22 +166,6 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
function pause(): void {
|
||||
if (player && isPlaying && !isPaused) {
|
||||
player.pause();
|
||||
setIsPaused(true);
|
||||
console.log('[AudioPlayer] Playback paused');
|
||||
}
|
||||
}
|
||||
|
||||
function resume(): void {
|
||||
if (player && isPaused) {
|
||||
player.play();
|
||||
setIsPaused(false);
|
||||
console.log('[AudioPlayer] Playback resumed');
|
||||
}
|
||||
}
|
||||
|
||||
function clearQueue(): void {
|
||||
// Reject all pending promises in the queue
|
||||
while (queueRef.current.length > 0) {
|
||||
@@ -250,10 +177,7 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
return {
|
||||
play,
|
||||
stop,
|
||||
pause,
|
||||
resume,
|
||||
isPlaying: () => isPlaying,
|
||||
isPaused: () => isPaused,
|
||||
clearQueue,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useAudioRecorder as useExpoAudioRecorder, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
|
||||
import { useAudioRecorder as useExpoAudioRecorder, useAudioRecorderState, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
|
||||
import { Paths, File, Directory, FileInfo } from 'expo-file-system';
|
||||
import { useState, useEffect } from 'react';
|
||||
|
||||
@@ -159,24 +159,25 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
};
|
||||
|
||||
const audioRecorder = useExpoAudioRecorder(recordingOptions);
|
||||
const recorderState = useAudioRecorderState(audioRecorder, 100);
|
||||
|
||||
// Monitor audio levels if metering is enabled
|
||||
useEffect(() => {
|
||||
if (!config?.onAudioLevel || !isRecording) return;
|
||||
|
||||
const interval = setInterval(() => {
|
||||
const metering = audioRecorder.state?.metering;
|
||||
const metering = recorderState.metering;
|
||||
if (metering !== undefined && metering !== null) {
|
||||
// Normalize metering value (typically ranges from -160 to 0 dB)
|
||||
// Convert to 0-1 range where 0 is silence and 1 is loud
|
||||
// We'll use -40 dB as the threshold for "loud"
|
||||
const normalized = Math.max(0, Math.min(1, (metering + 40) / 40));
|
||||
config.onAudioLevel(normalized);
|
||||
config.onAudioLevel?.(normalized);
|
||||
}
|
||||
}, 100); // Check every 100ms
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [isRecording, config, audioRecorder.state?.metering]);
|
||||
}, [isRecording, config, recorderState.metering]);
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (isRecording) {
|
||||
|
||||
@@ -10,6 +10,7 @@ export interface RealtimeAudioConfig {
|
||||
onError?: (error: Error) => void;
|
||||
volumeThreshold?: number; // RMS threshold for speech detection (0-1)
|
||||
silenceDuration?: number; // ms of silence before ending segment
|
||||
speechStartDuration?: number; // ms of sustained volume before starting segment
|
||||
}
|
||||
|
||||
export interface RealtimeAudio {
|
||||
@@ -19,6 +20,115 @@ export interface RealtimeAudio {
|
||||
isSpeaking: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert raw PCM data to WAV format by adding WAV headers
|
||||
* @param pcmBase64 Base64-encoded PCM data
|
||||
* @param sampleRate Sample rate (default: 16000)
|
||||
* @param channels Number of channels (default: 1 for mono)
|
||||
* @param bitsPerSample Bits per sample (default: 16)
|
||||
* @returns Base64-encoded WAV data
|
||||
*/
|
||||
function pcmToWav(
|
||||
pcmBase64: string,
|
||||
sampleRate: number = 16000,
|
||||
channels: number = 1,
|
||||
bitsPerSample: number = 16
|
||||
): string {
|
||||
// Decode base64 PCM data
|
||||
const pcmBinary = atob(pcmBase64);
|
||||
const pcmLength = pcmBinary.length;
|
||||
|
||||
// Calculate WAV parameters
|
||||
const byteRate = sampleRate * channels * (bitsPerSample / 8);
|
||||
const blockAlign = channels * (bitsPerSample / 8);
|
||||
const dataSize = pcmLength;
|
||||
const fileSize = 44 + dataSize; // WAV header is 44 bytes
|
||||
|
||||
// Create WAV buffer
|
||||
const wavBuffer = new Uint8Array(fileSize);
|
||||
const view = new DataView(wavBuffer.buffer);
|
||||
|
||||
// Write WAV header
|
||||
let offset = 0;
|
||||
|
||||
// RIFF chunk descriptor
|
||||
writeString(view, offset, 'RIFF'); offset += 4;
|
||||
view.setUint32(offset, fileSize - 8, true); offset += 4; // File size - 8
|
||||
writeString(view, offset, 'WAVE'); offset += 4;
|
||||
|
||||
// fmt sub-chunk
|
||||
writeString(view, offset, 'fmt '); offset += 4;
|
||||
view.setUint32(offset, 16, true); offset += 4; // Subchunk size (16 for PCM)
|
||||
view.setUint16(offset, 1, true); offset += 2; // Audio format (1 = PCM)
|
||||
view.setUint16(offset, channels, true); offset += 2;
|
||||
view.setUint32(offset, sampleRate, true); offset += 4;
|
||||
view.setUint32(offset, byteRate, true); offset += 4;
|
||||
view.setUint16(offset, blockAlign, true); offset += 2;
|
||||
view.setUint16(offset, bitsPerSample, true); offset += 2;
|
||||
|
||||
// data sub-chunk
|
||||
writeString(view, offset, 'data'); offset += 4;
|
||||
view.setUint32(offset, dataSize, true); offset += 4;
|
||||
|
||||
// Copy PCM data
|
||||
for (let i = 0; i < pcmLength; i++) {
|
||||
wavBuffer[offset + i] = pcmBinary.charCodeAt(i);
|
||||
}
|
||||
|
||||
// Convert to base64
|
||||
let binary = '';
|
||||
for (let i = 0; i < wavBuffer.length; i++) {
|
||||
binary += String.fromCharCode(wavBuffer[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to write string to DataView
|
||||
*/
|
||||
function writeString(view: DataView, offset: number, str: string): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode base64 string to Uint8Array
|
||||
*/
|
||||
function decodeBase64ToUint8Array(base64: string): Uint8Array {
|
||||
const binary = atob(base64);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode Uint8Array to base64 string
|
||||
*/
|
||||
function uint8ArrayToBase64(bytes: Uint8Array): string {
|
||||
let binary = '';
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
/**
|
||||
* Concatenate multiple Uint8Arrays into one
|
||||
*/
|
||||
function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
|
||||
const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
|
||||
const result = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const arr of arrays) {
|
||||
result.set(arr, offset);
|
||||
offset += arr.length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for realtime audio streaming with volume-based VAD
|
||||
* Uses expo-audio-studio for recording with echo cancellation and noise suppression
|
||||
@@ -28,13 +138,17 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
|
||||
const audioRecorder = useAudioRecorder();
|
||||
const audioBufferRef = useRef<string[]>([]);
|
||||
const audioBufferRef = useRef<Uint8Array[]>([]);
|
||||
const silenceStartRef = useRef<number | null>(null);
|
||||
const isSpeakingRef = useRef(false);
|
||||
const speechStartRef = useRef<number | null>(null); // Track when volume first exceeded threshold
|
||||
const speechConfirmedRef = useRef(false); // Track if speech has been confirmed
|
||||
const lastVolumeCheckRef = useRef<number>(Date.now());
|
||||
const lastVolumeLogRef = useRef<number>(Date.now());
|
||||
|
||||
const VOLUME_THRESHOLD = config.volumeThreshold ?? 0.02;
|
||||
const VOLUME_THRESHOLD = config.volumeThreshold ?? 0.2;
|
||||
const SILENCE_DURATION_MS = config.silenceDuration ?? 1000;
|
||||
const SPEECH_START_DURATION_MS = config.speechStartDuration ?? 1000;
|
||||
const VOLUME_CHECK_INTERVAL = 200; // Check volume every 200ms
|
||||
|
||||
async function requestMicrophonePermission(): Promise<boolean> {
|
||||
@@ -84,7 +198,6 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
|
||||
}
|
||||
|
||||
const rms = Math.sqrt(sum / sampleCount);
|
||||
console.log('[RealtimeAudio] Volume:', rms.toFixed(6));
|
||||
return rms;
|
||||
} catch (error) {
|
||||
console.error('[RealtimeAudio] Volume calculation error:', error);
|
||||
@@ -126,38 +239,84 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
|
||||
const volume = calculateVolume(audioData);
|
||||
const speechDetected = volume > VOLUME_THRESHOLD;
|
||||
|
||||
if (speechDetected && !isSpeakingRef.current) {
|
||||
// Speech START
|
||||
console.log('[RealtimeAudio] Speech started (volume:', volume.toFixed(4), ')');
|
||||
isSpeakingRef.current = true;
|
||||
// Decode base64 chunk to binary for buffering
|
||||
const decodedChunk = decodeBase64ToUint8Array(audioData);
|
||||
|
||||
// Check if we're in detection phase (volume detected but not yet confirmed)
|
||||
const inDetectionPhase = speechStartRef.current !== null && !isSpeakingRef.current;
|
||||
|
||||
if (inDetectionPhase) {
|
||||
// We're in detection phase - check timer regardless of current volume
|
||||
const sustainedDuration = Date.now() - speechStartRef.current!;
|
||||
|
||||
if (sustainedDuration >= SPEECH_START_DURATION_MS) {
|
||||
// Timer elapsed - confirm speech
|
||||
console.log('[RealtimeAudio] ✅ Speech confirmed after', sustainedDuration, 'ms (volume:', volume.toFixed(4), 'threshold:', VOLUME_THRESHOLD, ')');
|
||||
isSpeakingRef.current = true;
|
||||
speechConfirmedRef.current = true;
|
||||
setIsSpeaking(true);
|
||||
config.onSpeechStart?.();
|
||||
audioBufferRef.current.push(decodedChunk);
|
||||
silenceStartRef.current = null;
|
||||
} else if (speechDetected) {
|
||||
// Still in detection phase, volume still above threshold, keep buffering
|
||||
audioBufferRef.current.push(decodedChunk);
|
||||
|
||||
// Log progress every 500ms during detection
|
||||
const logNow = Date.now();
|
||||
if (logNow - lastVolumeLogRef.current >= 500) {
|
||||
console.log('[RealtimeAudio] 🔍 Detecting...', sustainedDuration, '/', SPEECH_START_DURATION_MS, 'ms (volume:', volume.toFixed(4), ')');
|
||||
lastVolumeLogRef.current = logNow;
|
||||
}
|
||||
} else {
|
||||
// Volume dropped before timer elapsed - false start
|
||||
console.log('[RealtimeAudio] ❌ False start (volume dropped after', sustainedDuration, 'ms, needed', SPEECH_START_DURATION_MS, 'ms)');
|
||||
speechStartRef.current = null;
|
||||
speechConfirmedRef.current = false;
|
||||
audioBufferRef.current = [];
|
||||
}
|
||||
} else if (speechDetected && !isSpeakingRef.current) {
|
||||
// First volume detection - start timer and buffering
|
||||
console.log('[RealtimeAudio] 🎤 Volume spike detected! Starting detection... (volume:', volume.toFixed(4), 'threshold:', VOLUME_THRESHOLD, ')');
|
||||
speechStartRef.current = Date.now();
|
||||
speechConfirmedRef.current = false;
|
||||
audioBufferRef.current = [decodedChunk];
|
||||
silenceStartRef.current = null;
|
||||
audioBufferRef.current = [audioData];
|
||||
setIsSpeaking(true);
|
||||
config.onSpeechStart?.();
|
||||
} else if (speechDetected && isSpeakingRef.current) {
|
||||
// Continuing speech
|
||||
audioBufferRef.current.push(audioData);
|
||||
// Continuing confirmed speech - just buffer, no logs
|
||||
audioBufferRef.current.push(decodedChunk);
|
||||
silenceStartRef.current = null;
|
||||
} else if (!speechDetected && isSpeakingRef.current) {
|
||||
// Potential speech END
|
||||
audioBufferRef.current.push(audioData);
|
||||
// Potential speech END (after confirmed speech)
|
||||
audioBufferRef.current.push(decodedChunk);
|
||||
|
||||
if (silenceStartRef.current === null) {
|
||||
silenceStartRef.current = Date.now();
|
||||
console.log('[RealtimeAudio] 🔇 Silence detected, waiting', SILENCE_DURATION_MS, 'ms to confirm end...');
|
||||
} else {
|
||||
const silenceDuration = Date.now() - silenceStartRef.current;
|
||||
if (silenceDuration >= SILENCE_DURATION_MS) {
|
||||
// Speech END confirmed
|
||||
console.log('[RealtimeAudio] Speech ended after', silenceDuration, 'ms silence');
|
||||
const totalChunks = audioBufferRef.current.length;
|
||||
console.log('[RealtimeAudio] 🛑 Speech ended after', silenceDuration, 'ms silence (captured', totalChunks, 'chunks)');
|
||||
isSpeakingRef.current = false;
|
||||
speechStartRef.current = null;
|
||||
speechConfirmedRef.current = false;
|
||||
silenceStartRef.current = null;
|
||||
setIsSpeaking(false);
|
||||
config.onSpeechEnd?.();
|
||||
|
||||
// Send buffered audio segment
|
||||
if (audioBufferRef.current.length > 0) {
|
||||
const combinedAudio = audioBufferRef.current.join('');
|
||||
config.onAudioSegment?.(combinedAudio);
|
||||
// Concatenate all binary chunks
|
||||
const combinedBinary = concatenateUint8Arrays(audioBufferRef.current);
|
||||
const audioSizeKb = (combinedBinary.length / 1024).toFixed(2);
|
||||
console.log('[RealtimeAudio] 📤 Sending audio segment:', audioSizeKb, 'KB');
|
||||
// Convert to base64
|
||||
const combinedPcmBase64 = uint8ArrayToBase64(combinedBinary);
|
||||
// Convert PCM to WAV format
|
||||
const wavAudio = pcmToWav(combinedPcmBase64, 16000, 1, 16);
|
||||
config.onAudioSegment?.(wavAudio);
|
||||
audioBufferRef.current = [];
|
||||
}
|
||||
}
|
||||
@@ -165,13 +324,14 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
|
||||
}
|
||||
} else if (isSpeakingRef.current) {
|
||||
// Just buffer if we're speaking but not checking volume yet
|
||||
audioBufferRef.current.push(audioData);
|
||||
const decodedChunk = decodeBase64ToUint8Array(audioData);
|
||||
audioBufferRef.current.push(decodedChunk);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
setIsActive(true);
|
||||
console.log('[RealtimeAudio] Realtime audio started successfully');
|
||||
console.log('[RealtimeAudio] 🎙️ Audio capture started (threshold:', VOLUME_THRESHOLD, 'duration:', SPEECH_START_DURATION_MS, 'ms)');
|
||||
} catch (error) {
|
||||
console.error('[RealtimeAudio] Start error:', error);
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
@@ -198,6 +358,8 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio {
|
||||
// Reset state
|
||||
audioBufferRef.current = [];
|
||||
isSpeakingRef.current = false;
|
||||
speechStartRef.current = null;
|
||||
speechConfirmedRef.current = false;
|
||||
silenceStartRef.current = null;
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
|
||||
@@ -0,0 +1,352 @@
|
||||
import { useState, useEffect, useRef, useCallback } from "react";
|
||||
import {
|
||||
initialize,
|
||||
toggleRecording,
|
||||
tearDown,
|
||||
useExpoTwoWayAudioEventListener,
|
||||
type MicrophoneDataCallback,
|
||||
type VolumeLevelCallback,
|
||||
} from "@speechmatics/expo-two-way-audio";
|
||||
|
||||
export interface SpeechmaticsAudioConfig {
|
||||
onAudioSegment?: (audioData: string) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
volumeThreshold: number; // Volume threshold for speech detection (0-1)
|
||||
silenceDuration: number; // ms of silence before ending segment
|
||||
speechConfirmationDuration: number; // ms of sustained speech before confirming
|
||||
detectionGracePeriod: number; // ms grace period for volume dips during detection
|
||||
}
|
||||
|
||||
export interface SpeechmaticsAudio {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
isActive: boolean;
|
||||
isSpeaking: boolean;
|
||||
isDetecting: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert raw PCM data to WAV format by adding WAV headers
|
||||
*/
|
||||
function pcmToWav(
|
||||
pcmBase64: string,
|
||||
sampleRate: number = 16000,
|
||||
channels: number = 1,
|
||||
bitsPerSample: number = 16
|
||||
): string {
|
||||
const pcmBinary = atob(pcmBase64);
|
||||
const pcmLength = pcmBinary.length;
|
||||
|
||||
const byteRate = sampleRate * channels * (bitsPerSample / 8);
|
||||
const blockAlign = channels * (bitsPerSample / 8);
|
||||
const dataSize = pcmLength;
|
||||
const fileSize = 44 + dataSize;
|
||||
|
||||
const wavBuffer = new Uint8Array(fileSize);
|
||||
const view = new DataView(wavBuffer.buffer);
|
||||
|
||||
let offset = 0;
|
||||
|
||||
// RIFF chunk descriptor
|
||||
writeString(view, offset, "RIFF");
|
||||
offset += 4;
|
||||
view.setUint32(offset, fileSize - 8, true);
|
||||
offset += 4;
|
||||
writeString(view, offset, "WAVE");
|
||||
offset += 4;
|
||||
|
||||
// fmt sub-chunk
|
||||
writeString(view, offset, "fmt ");
|
||||
offset += 4;
|
||||
view.setUint32(offset, 16, true);
|
||||
offset += 4;
|
||||
view.setUint16(offset, 1, true);
|
||||
offset += 2;
|
||||
view.setUint16(offset, channels, true);
|
||||
offset += 2;
|
||||
view.setUint32(offset, sampleRate, true);
|
||||
offset += 4;
|
||||
view.setUint32(offset, byteRate, true);
|
||||
offset += 4;
|
||||
view.setUint16(offset, blockAlign, true);
|
||||
offset += 2;
|
||||
view.setUint16(offset, bitsPerSample, true);
|
||||
offset += 2;
|
||||
|
||||
// data sub-chunk
|
||||
writeString(view, offset, "data");
|
||||
offset += 4;
|
||||
view.setUint32(offset, dataSize, true);
|
||||
offset += 4;
|
||||
|
||||
// Copy PCM data
|
||||
for (let i = 0; i < pcmLength; i++) {
|
||||
wavBuffer[offset + i] = pcmBinary.charCodeAt(i);
|
||||
}
|
||||
|
||||
// Convert to base64
|
||||
let binary = "";
|
||||
for (let i = 0; i < wavBuffer.length; i++) {
|
||||
binary += String.fromCharCode(wavBuffer[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function writeString(view: DataView, offset: number, str: string): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
function uint8ArrayToBase64(bytes: Uint8Array): string {
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
|
||||
const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
|
||||
const result = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const arr of arrays) {
|
||||
result.set(arr, offset);
|
||||
offset += arr.length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio
|
||||
*/
|
||||
export function useSpeechmaticsAudio(
|
||||
config: SpeechmaticsAudioConfig
|
||||
): SpeechmaticsAudio {
|
||||
const [isActive, setIsActive] = useState(false);
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
const [isDetecting, setIsDetecting] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
|
||||
const audioBufferRef = useRef<Uint8Array[]>([]);
|
||||
const silenceStartRef = useRef<number | null>(null);
|
||||
const isSpeakingRef = useRef(false);
|
||||
const speechDetectionStartRef = useRef<number | null>(null);
|
||||
const speechConfirmedRef = useRef(false);
|
||||
const detectionSilenceStartRef = useRef<number | null>(null);
|
||||
|
||||
const VOLUME_THRESHOLD = config.volumeThreshold;
|
||||
const SILENCE_DURATION_MS = config.silenceDuration;
|
||||
const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration;
|
||||
const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod;
|
||||
|
||||
// Initialize audio on mount
|
||||
useEffect(() => {
|
||||
const initializeAudio = async () => {
|
||||
try {
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
console.log("[SpeechmaticsAudio] Audio initialized");
|
||||
} catch (error) {
|
||||
console.error("[SpeechmaticsAudio] Initialization error:", error);
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
}
|
||||
};
|
||||
initializeAudio();
|
||||
|
||||
return () => {
|
||||
tearDown();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Listen to microphone data
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
useCallback<MicrophoneDataCallback>(
|
||||
(event) => {
|
||||
if (!isActive) return;
|
||||
|
||||
const pcmData: Uint8Array = event.data;
|
||||
|
||||
// Buffer the audio chunk if we're detecting or speaking
|
||||
// Start buffering from first spike to capture beginning of speech
|
||||
if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
|
||||
audioBufferRef.current.push(pcmData);
|
||||
}
|
||||
},
|
||||
[isActive]
|
||||
)
|
||||
);
|
||||
|
||||
// Listen to volume level for VAD
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
useCallback<VolumeLevelCallback>(
|
||||
(event) => {
|
||||
if (!isActive) return;
|
||||
|
||||
const volume: number = event.data;
|
||||
const speechDetected = volume > VOLUME_THRESHOLD;
|
||||
|
||||
// console.log('[SpeechmaticsAudio] Volume:', volume.toFixed(6));
|
||||
|
||||
if (speechDetected && !isSpeakingRef.current && !speechConfirmedRef.current) {
|
||||
// Initial speech detection - start tracking
|
||||
if (speechDetectionStartRef.current === null) {
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech started (volume:",
|
||||
volume.toFixed(4),
|
||||
")"
|
||||
);
|
||||
speechDetectionStartRef.current = Date.now();
|
||||
detectionSilenceStartRef.current = null;
|
||||
audioBufferRef.current = [];
|
||||
setIsDetecting(true);
|
||||
} else {
|
||||
// Volume is back above threshold - reset grace period
|
||||
detectionSilenceStartRef.current = null;
|
||||
|
||||
// Check if speech has been sustained long enough
|
||||
const speechDuration = Date.now() - speechDetectionStartRef.current;
|
||||
if (speechDuration >= SPEECH_CONFIRMATION_MS) {
|
||||
// Speech CONFIRMED
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech confirmed after",
|
||||
speechDuration,
|
||||
"ms"
|
||||
);
|
||||
isSpeakingRef.current = true;
|
||||
speechConfirmedRef.current = true;
|
||||
silenceStartRef.current = null;
|
||||
setIsDetecting(false);
|
||||
setIsSpeaking(true);
|
||||
config.onSpeechStart?.();
|
||||
}
|
||||
}
|
||||
} else if (speechDetected && isSpeakingRef.current && speechConfirmedRef.current) {
|
||||
// Continuing confirmed speech
|
||||
silenceStartRef.current = null;
|
||||
} else if (!speechDetected && !speechConfirmedRef.current && speechDetectionStartRef.current !== null) {
|
||||
// Volume dropped during detection phase - apply grace period
|
||||
if (detectionSilenceStartRef.current === null) {
|
||||
detectionSilenceStartRef.current = Date.now();
|
||||
} else {
|
||||
const graceDuration = Date.now() - detectionSilenceStartRef.current;
|
||||
if (graceDuration >= DETECTION_GRACE_PERIOD_MS) {
|
||||
// Grace period expired - cancel detection
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech detection cancelled after",
|
||||
graceDuration,
|
||||
"ms grace period"
|
||||
);
|
||||
speechDetectionStartRef.current = null;
|
||||
detectionSilenceStartRef.current = null;
|
||||
audioBufferRef.current = [];
|
||||
setIsDetecting(false);
|
||||
}
|
||||
}
|
||||
} else if (!speechDetected && isSpeakingRef.current && speechConfirmedRef.current) {
|
||||
// Potential speech END
|
||||
if (silenceStartRef.current === null) {
|
||||
silenceStartRef.current = Date.now();
|
||||
} else {
|
||||
const silenceDuration = Date.now() - silenceStartRef.current;
|
||||
if (silenceDuration >= SILENCE_DURATION_MS) {
|
||||
// Speech END confirmed
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech ended after",
|
||||
silenceDuration,
|
||||
"ms silence"
|
||||
);
|
||||
isSpeakingRef.current = false;
|
||||
speechConfirmedRef.current = false;
|
||||
speechDetectionStartRef.current = null;
|
||||
silenceStartRef.current = null;
|
||||
setIsSpeaking(false);
|
||||
config.onSpeechEnd?.();
|
||||
|
||||
// Send buffered audio segment
|
||||
if (audioBufferRef.current.length > 0) {
|
||||
const combinedBinary = concatenateUint8Arrays(
|
||||
audioBufferRef.current
|
||||
);
|
||||
const combinedPcmBase64 = uint8ArrayToBase64(combinedBinary);
|
||||
const wavAudio = pcmToWav(combinedPcmBase64, 16000, 1, 16);
|
||||
config.onAudioSegment?.(wavAudio);
|
||||
audioBufferRef.current = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
[isActive, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config]
|
||||
)
|
||||
);
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (!audioInitialized) {
|
||||
throw new Error("Audio not initialized");
|
||||
}
|
||||
|
||||
if (isActive) {
|
||||
console.log("[SpeechmaticsAudio] Already active");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
console.log("[SpeechmaticsAudio] Starting audio capture...");
|
||||
|
||||
// Start recording
|
||||
toggleRecording(true);
|
||||
|
||||
setIsActive(true);
|
||||
console.log("[SpeechmaticsAudio] Audio capture started successfully");
|
||||
} catch (error) {
|
||||
console.error("[SpeechmaticsAudio] Start error:", error);
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
await stop();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<void> {
|
||||
console.log("[SpeechmaticsAudio] Stopping audio capture...");
|
||||
|
||||
// Stop recording
|
||||
toggleRecording(false);
|
||||
|
||||
// Reset state
|
||||
audioBufferRef.current = [];
|
||||
isSpeakingRef.current = false;
|
||||
speechConfirmedRef.current = false;
|
||||
speechDetectionStartRef.current = null;
|
||||
detectionSilenceStartRef.current = null;
|
||||
silenceStartRef.current = null;
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
|
||||
console.log("[SpeechmaticsAudio] Audio capture stopped");
|
||||
}
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (isActive) {
|
||||
stop();
|
||||
}
|
||||
};
|
||||
}, [isActive]);
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
isActive,
|
||||
isSpeaking,
|
||||
isDetecting,
|
||||
};
|
||||
}
|
||||
@@ -58,7 +58,7 @@ export function useWebSocket(url: string, conversationId?: string | null): UseWe
|
||||
// Only session messages trigger handlers
|
||||
if (wsMessage.type === 'session') {
|
||||
const sessionMessage = wsMessage.message;
|
||||
console.log(`[WS] Received session message type: ${sessionMessage.type}`);
|
||||
// console.log(`[WS] Received session message type: ${sessionMessage.type}`);
|
||||
|
||||
// Track conversation ID when loaded
|
||||
if (sessionMessage.type === 'conversation_loaded') {
|
||||
|
||||
Reference in New Issue
Block a user