jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: migrate to @boudra/expo-two-way-audio and add tool call details bottom sheet

Browse Source
This commit is contained in:
Mohamed Boudra
2025-10-23 20:01:52 +02:00
parent 74fffcd8a3
commit 15320819a7
9 changed files with 417 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/hooks/use-audio-player.ts
+56 -15
View File
@@ -1,5 +1,11 @@
import { useState, useRef } from 'react';
import { initialize, playPCMData } from '@speechmatics/expo-two-way-audio';
import { useState, useRef } from "react";
import {
initialize,
playPCMData,
stopPlayback,
pausePlayback,
resumePlayback,
} from "@boudra/expo-two-way-audio";
interface QueuedAudio {
audioData: Blob;
@@ -14,7 +20,7 @@ interface QueuedAudio {
function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array {
// PCM16 = 2 bytes per sample
const samples24k = pcm24k.length / 2;
const samples16k = Math.floor(samples24k * 16000 / 24000);
const samples16k = Math.floor((samples24k * 16000) / 24000);
const pcm16k = new Uint8Array(samples16k * 2);
const ratio = 24000 / 16000; // 1.5
@@ -27,10 +33,10 @@ function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array {
}
}
console.log('[AudioPlayer] Resampled PCM:', {
console.log("[AudioPlayer] Resampled PCM:", {
input24k: pcm24k.length,
output16k: pcm16k.length,
durationMs: (samples16k / 16),
durationMs: samples16k / 16,
});
return pcm16k;
@@ -44,6 +50,7 @@ export function useAudioPlayer() {
const [audioInitialized, setAudioInitialized] = useState(false);
const queueRef = useRef<QueuedAudio[]>([]);
const isProcessingQueueRef = useRef(false);
const playbackTimeoutRef = useRef<NodeJS.Timeout | null>(null);
async function play(audioData: Blob): Promise<number> {
return new Promise((resolve, reject) => {
@@ -100,12 +107,19 @@ export function useAudioPlayer() {
// Initialize audio if not already initialized
if (!audioInitialized) {
console.log('[AudioPlayer] Initializing audio...');
console.log("[AudioPlayer] Initializing audio...");
await initialize();
setAudioInitialized(true);
console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)');
console.log(
"[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)"
);
}
// Workaround: Resume playback before playing new audio to ensure the audio engine is ready
// This fixes the issue where playback doesn't work after calling stopPlayback()
console.log("[AudioPlayer] Resuming playback engine...");
resumePlayback();
// Get PCM data from blob (server now sends PCM format)
const arrayBuffer = await audioData.arrayBuffer();
let pcm24k = new Uint8Array(arrayBuffer);
@@ -118,22 +132,40 @@ export function useAudioPlayer() {
const durationSec = samples / 16000; // 16kHz sample rate
const audioSizeKb = (pcm16k.length / 1024).toFixed(2);
console.log('[AudioPlayer] 🔊 Playing audio:', audioSizeKb, 'KB, duration:', durationSec.toFixed(2), 's');
console.log(
"[AudioPlayer] 🔊 Playing audio:",
audioSizeKb,
"KB, duration:",
durationSec.toFixed(2),
"s"
);
setIsPlaying(true);
// Play entire PCM data at once through Speechmatics
playPCMData(pcm16k);
// Clear any existing timeout
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
}
// Wait for playback to finish (estimate based on duration)
setTimeout(() => {
console.log('[AudioPlayer] ✅ Playback finished');
playbackTimeoutRef.current = setTimeout(() => {
console.log("[AudioPlayer] ✅ Playback finished");
setIsPlaying(false);
playbackTimeoutRef.current = null;
resolve(durationSec);
}, durationSec * 1000);
} catch (error) {
console.error('[AudioPlayer] Error playing audio:', error);
console.error("[AudioPlayer] Error playing audio:", error);
// Clear timeout on error
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
playbackTimeoutRef.current = null;
}
setIsPlaying(false);
reject(error);
}
@@ -142,7 +174,16 @@ export function useAudioPlayer() {
function stop(): void {
if (isPlaying) {
console.log('[AudioPlayer] 🛑 Stopping playback (interrupted)');
console.log("[AudioPlayer] 🛑 Stopping playback (interrupted)");
// Stop native playback
stopPlayback();
// Clear playback timeout
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
playbackTimeoutRef.current = null;
}
}
setIsPlaying(false);
@@ -150,7 +191,7 @@ export function useAudioPlayer() {
// Reject all pending promises in the queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error('Playback stopped'));
item.reject(new Error("Playback stopped"));
}
isProcessingQueueRef.current = false;
@@ -160,7 +201,7 @@ export function useAudioPlayer() {
// Reject all pending promises in the queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error('Queue cleared'));
item.reject(new Error("Queue cleared"));
}
}
packages/app/src/hooks/use-speechmatics-audio.ts
+30 -6
View File
@@ -6,7 +6,7 @@ import {
useExpoTwoWayAudioEventListener,
type MicrophoneDataCallback,
type VolumeLevelCallback,
} from "@speechmatics/expo-two-way-audio";
} from "@boudra/expo-two-way-audio";
export interface SpeechmaticsAudioConfig {
onAudioSegment?: (audioData: string) => void;
@@ -200,7 +200,11 @@ export function useSpeechmaticsAudio(
// console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD);
if (speechDetected && !isSpeakingRef.current && !speechConfirmedRef.current) {
if (
speechDetected &&
!isSpeakingRef.current &&
!speechConfirmedRef.current
) {
// Initial speech detection - start tracking
if (speechDetectionStartRef.current === null) {
console.log(
@@ -233,10 +237,18 @@ export function useSpeechmaticsAudio(
config.onSpeechStart?.();
}
}
} else if (speechDetected && isSpeakingRef.current && speechConfirmedRef.current) {
} else if (
speechDetected &&
isSpeakingRef.current &&
speechConfirmedRef.current
) {
// Continuing confirmed speech
silenceStartRef.current = null;
} else if (!speechDetected && !speechConfirmedRef.current && speechDetectionStartRef.current !== null) {
} else if (
!speechDetected &&
!speechConfirmedRef.current &&
speechDetectionStartRef.current !== null
) {
// Volume dropped during detection phase - apply grace period
if (detectionSilenceStartRef.current === null) {
detectionSilenceStartRef.current = Date.now();
@@ -255,7 +267,11 @@ export function useSpeechmaticsAudio(
setIsDetecting(false);
}
}
} else if (!speechDetected && isSpeakingRef.current && speechConfirmedRef.current) {
} else if (
!speechDetected &&
isSpeakingRef.current &&
speechConfirmedRef.current
) {
// Potential speech END
if (silenceStartRef.current === null) {
silenceStartRef.current = Date.now();
@@ -289,7 +305,15 @@ export function useSpeechmaticsAudio(
}
}
},
[isActive, isMuted, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config]
[
isActive,
isMuted,
VOLUME_THRESHOLD,
SILENCE_DURATION_MS,
SPEECH_CONFIRMATION_MS,
DETECTION_GRACE_PERIOD_MS,
config,
]
)
);