jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix web voice mode; share VAD/segmenting

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-04 08:22:27 +07:00
parent f08efd2da6
commit 363566587e
7 changed files with 1041 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/hooks/use-speechmatics-audio.native.ts
+67 -225
View File
@@ -1,5 +1,4 @@
import { useState, useEffect, useRef, useCallback } from "react";
import { Buffer } from "buffer";
import {
initialize,
useMicrophonePermissions,
@@ -10,6 +9,8 @@ import {
type VolumeLevelCallback,
} from "@boudra/expo-two-way-audio";
import { SpeechSegmenter } from "@/voice/speech-segmenter";
export interface SpeechmaticsAudioConfig {
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
onSpeechStart?: () => void;
@@ -35,31 +36,6 @@ export interface SpeechmaticsAudio {
segmentDuration: number;
}
function uint8ArrayToBase64(bytes: Uint8Array): string {
// NOTE: This is performance-sensitive during continuous streaming.
// Buffer-backed base64 is significantly faster than manual string building.
try {
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
} catch {
let binary = "";
for (let i = 0; i < bytes.length; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
}
}
function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
const result = new Uint8Array(totalLength);
let offset = 0;
for (const arr of arrays) {
result.set(arr, offset);
offset += arr.length;
}
return result;
}
/**
* Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio
*/
@@ -78,49 +54,64 @@ export function useSpeechmaticsAudio(
const enableContinuousStreaming = config.enableContinuousStreaming === true;
const audioBufferRef = useRef<Uint8Array[]>([]);
const silenceStartRef = useRef<number | null>(null);
const isSpeakingRef = useRef(false);
const speechDetectionStartRef = useRef<number | null>(null);
const speechConfirmedRef = useRef(false);
const detectionSilenceStartRef = useRef<number | null>(null);
const bufferedBytesRef = useRef(0);
const isActiveRef = useRef(isActive);
useEffect(() => {
isActiveRef.current = isActive;
}, [isActive]);
const PCM_SAMPLE_RATE = 16000;
const PCM_CHANNELS = 1;
const PCM_BITS_PER_SAMPLE = 16;
const PCM_BYTES_PER_MS =
(PCM_SAMPLE_RATE * PCM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8)) / 1000;
const MIN_CHUNK_DURATION_MS = 1000;
const MIN_CHUNK_BYTES = Math.round(
PCM_BYTES_PER_MS * MIN_CHUNK_DURATION_MS
);
const isMutedRef = useRef(isMuted);
useEffect(() => {
isMutedRef.current = isMuted;
}, [isMuted]);
const flushBufferedAudio = useCallback(
(isLast: boolean) => {
if (audioBufferRef.current.length === 0) {
bufferedBytesRef.current = 0;
return;
const callbacksRef = useRef({
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
});
useEffect(() => {
callbacksRef.current = {
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
};
}, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd]);
const segmenterRef = useRef<SpeechSegmenter | null>(null);
if (segmenterRef.current === null) {
segmenterRef.current = new SpeechSegmenter(
{
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
},
{
onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
onDetectingChange: (next) => setIsDetecting(next),
onSpeakingChange: (next) => setIsSpeaking(next),
}
);
}
const combinedBinary = concatenateUint8Arrays(audioBufferRef.current);
const pcmBase64 = uint8ArrayToBase64(combinedBinary);
config.onAudioSegment?.({
audioData: pcmBase64,
isLast,
});
audioBufferRef.current = [];
bufferedBytesRef.current = 0;
},
[config]
);
const VOLUME_THRESHOLD = config.volumeThreshold;
const SILENCE_DURATION_MS = config.silenceDuration;
const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration;
const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod;
useEffect(() => {
segmenterRef.current?.updateConfig({
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
});
}, [
enableContinuousStreaming,
config.volumeThreshold,
config.silenceDuration,
config.speechConfirmationDuration,
config.detectionGracePeriod,
]);
// Update segment duration timer
useEffect(() => {
@@ -129,7 +120,7 @@ export function useSpeechmaticsAudio(
return;
}
const startTime = speechDetectionStartRef.current || Date.now();
const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
const interval = setInterval(() => {
const elapsed = Date.now() - startTime;
setSegmentDuration(elapsed);
@@ -143,34 +134,12 @@ export function useSpeechmaticsAudio(
"onMicrophoneData",
useCallback<MicrophoneDataCallback>(
(event) => {
if (!isActive || isMuted) return;
if (!isActiveRef.current || isMutedRef.current) return;
const pcmData: Uint8Array = event.data;
if (enableContinuousStreaming) {
audioBufferRef.current.push(pcmData);
bufferedBytesRef.current += pcmData.length;
if (bufferedBytesRef.current >= MIN_CHUNK_BYTES) {
flushBufferedAudio(false);
}
return;
}
// Buffer the audio chunk if we're detecting or speaking
// Start buffering from first spike to capture beginning of speech
if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
audioBufferRef.current.push(pcmData);
bufferedBytesRef.current += pcmData.length;
if (
speechConfirmedRef.current &&
bufferedBytesRef.current >= MIN_CHUNK_BYTES
) {
flushBufferedAudio(false);
}
}
segmenterRef.current?.pushPcmChunk(pcmData);
},
[enableContinuousStreaming, isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
[]
)
);
@@ -179,128 +148,15 @@ export function useSpeechmaticsAudio(
"onInputVolumeLevelData",
useCallback<VolumeLevelCallback>(
(event) => {
if (!isActive) return;
if (!isActiveRef.current) return;
const volumeLevel: number = event.data;
setVolume(volumeLevel);
if (isMuted) return;
if (enableContinuousStreaming) return;
const speechDetected = volumeLevel > VOLUME_THRESHOLD;
// console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD);
if (
speechDetected &&
!isSpeakingRef.current &&
!speechConfirmedRef.current
) {
// Initial speech detection - start tracking
if (speechDetectionStartRef.current === null) {
console.log(
"[SpeechmaticsAudio] Speech started (volume:",
volumeLevel.toFixed(4),
")"
);
speechDetectionStartRef.current = Date.now();
detectionSilenceStartRef.current = null;
audioBufferRef.current = [];
bufferedBytesRef.current = 0;
setIsDetecting(true);
} else {
// Volume is back above threshold - reset grace period
detectionSilenceStartRef.current = null;
// Check if speech has been sustained long enough
const speechDuration = Date.now() - speechDetectionStartRef.current;
if (speechDuration >= SPEECH_CONFIRMATION_MS) {
// Speech CONFIRMED
console.log(
"[SpeechmaticsAudio] Speech confirmed after",
speechDuration,
"ms"
);
isSpeakingRef.current = true;
speechConfirmedRef.current = true;
silenceStartRef.current = null;
setIsDetecting(false);
setIsSpeaking(true);
config.onSpeechStart?.();
}
}
} else if (
speechDetected &&
isSpeakingRef.current &&
speechConfirmedRef.current
) {
// Continuing confirmed speech
silenceStartRef.current = null;
} else if (
!speechDetected &&
!speechConfirmedRef.current &&
speechDetectionStartRef.current !== null
) {
// Volume dropped during detection phase - apply grace period
if (detectionSilenceStartRef.current === null) {
detectionSilenceStartRef.current = Date.now();
} else {
const graceDuration = Date.now() - detectionSilenceStartRef.current;
if (graceDuration >= DETECTION_GRACE_PERIOD_MS) {
// Grace period expired - cancel detection
console.log(
"[SpeechmaticsAudio] Speech detection cancelled after",
graceDuration,
"ms grace period"
);
speechDetectionStartRef.current = null;
detectionSilenceStartRef.current = null;
audioBufferRef.current = [];
bufferedBytesRef.current = 0;
setIsDetecting(false);
}
}
} else if (
!speechDetected &&
isSpeakingRef.current &&
speechConfirmedRef.current
) {
// Potential speech END
if (silenceStartRef.current === null) {
silenceStartRef.current = Date.now();
} else {
const silenceDuration = Date.now() - silenceStartRef.current;
if (silenceDuration >= SILENCE_DURATION_MS) {
// Speech END confirmed
console.log(
"[SpeechmaticsAudio] Speech ended after",
silenceDuration,
"ms silence"
);
isSpeakingRef.current = false;
speechConfirmedRef.current = false;
speechDetectionStartRef.current = null;
silenceStartRef.current = null;
setIsSpeaking(false);
config.onSpeechEnd?.();
// Send buffered audio segment
flushBufferedAudio(true);
}
}
}
if (isMutedRef.current) return;
segmenterRef.current?.pushVolumeLevel(volumeLevel, Date.now());
},
[
enableContinuousStreaming,
isActive,
isMuted,
VOLUME_THRESHOLD,
SILENCE_DURATION_MS,
SPEECH_CONFIRMATION_MS,
DETECTION_GRACE_PERIOD_MS,
config,
flushBufferedAudio,
]
[]
)
);
@@ -363,9 +219,7 @@ export function useSpeechmaticsAudio(
toggleRecording(false);
}
if (enableContinuousStreaming) {
flushBufferedAudio(true);
}
segmenterRef.current?.stop(Date.now());
// Tear down audio session
if (audioInitialized) {
@@ -375,13 +229,7 @@ export function useSpeechmaticsAudio(
}
// Reset state
audioBufferRef.current = [];
bufferedBytesRef.current = 0;
isSpeakingRef.current = false;
speechConfirmedRef.current = false;
speechDetectionStartRef.current = null;
detectionSilenceStartRef.current = null;
silenceStartRef.current = null;
segmenterRef.current?.reset();
setIsActive(false);
setIsSpeaking(false);
setIsDetecting(false);
@@ -398,13 +246,7 @@ export function useSpeechmaticsAudio(
if (newMuted) {
// Clear any ongoing speech detection/speaking state
audioBufferRef.current = [];
bufferedBytesRef.current = 0;
isSpeakingRef.current = false;
speechConfirmedRef.current = false;
speechDetectionStartRef.current = null;
detectionSilenceStartRef.current = null;
silenceStartRef.current = null;
segmenterRef.current?.reset();
setIsSpeaking(false);
setIsDetecting(false);
}