feat: add git branch/worktree setup, message queueing, file explorer improvements, and server restart
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,401 @@
|
||||
import { useState, useEffect, useRef, useCallback } from "react";
|
||||
import {
|
||||
initialize,
|
||||
useMicrophonePermissions,
|
||||
toggleRecording,
|
||||
tearDown,
|
||||
useExpoTwoWayAudioEventListener,
|
||||
type MicrophoneDataCallback,
|
||||
type VolumeLevelCallback,
|
||||
} from "@boudra/expo-two-way-audio";
|
||||
|
||||
export interface SpeechmaticsAudioConfig {
|
||||
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
volumeThreshold: number; // Volume threshold for speech detection (0-1)
|
||||
silenceDuration: number; // ms of silence before ending segment
|
||||
speechConfirmationDuration: number; // ms of sustained speech before confirming
|
||||
detectionGracePeriod: number; // ms grace period for volume dips during detection
|
||||
}
|
||||
|
||||
export interface SpeechmaticsAudio {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
toggleMute: () => void;
|
||||
isActive: boolean;
|
||||
isSpeaking: boolean;
|
||||
isDetecting: boolean;
|
||||
isMuted: boolean;
|
||||
volume: number;
|
||||
segmentDuration: number;
|
||||
}
|
||||
|
||||
function uint8ArrayToBase64(bytes: Uint8Array): string {
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
|
||||
const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
|
||||
const result = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const arr of arrays) {
|
||||
result.set(arr, offset);
|
||||
offset += arr.length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio
|
||||
*/
|
||||
export function useSpeechmaticsAudio(
|
||||
config: SpeechmaticsAudioConfig
|
||||
): SpeechmaticsAudio {
|
||||
const [microphonePermission, requestMicrophonePermission] =
|
||||
useMicrophonePermissions();
|
||||
const [isActive, setIsActive] = useState(false);
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
const [isDetecting, setIsDetecting] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
const [volume, setVolume] = useState(0);
|
||||
const [isMuted, setIsMuted] = useState(false);
|
||||
const [segmentDuration, setSegmentDuration] = useState(0);
|
||||
|
||||
const audioBufferRef = useRef<Uint8Array[]>([]);
|
||||
const silenceStartRef = useRef<number | null>(null);
|
||||
const isSpeakingRef = useRef(false);
|
||||
const speechDetectionStartRef = useRef<number | null>(null);
|
||||
const speechConfirmedRef = useRef(false);
|
||||
const detectionSilenceStartRef = useRef<number | null>(null);
|
||||
const bufferedBytesRef = useRef(0);
|
||||
|
||||
const PCM_SAMPLE_RATE = 16000;
|
||||
const PCM_CHANNELS = 1;
|
||||
const PCM_BITS_PER_SAMPLE = 16;
|
||||
const PCM_BYTES_PER_MS =
|
||||
(PCM_SAMPLE_RATE * PCM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8)) / 1000;
|
||||
const MIN_CHUNK_DURATION_MS = 1000;
|
||||
const MIN_CHUNK_BYTES = Math.round(
|
||||
PCM_BYTES_PER_MS * MIN_CHUNK_DURATION_MS
|
||||
);
|
||||
|
||||
const flushBufferedAudio = useCallback(
|
||||
(isLast: boolean) => {
|
||||
if (audioBufferRef.current.length === 0) {
|
||||
bufferedBytesRef.current = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
const combinedBinary = concatenateUint8Arrays(audioBufferRef.current);
|
||||
const pcmBase64 = uint8ArrayToBase64(combinedBinary);
|
||||
|
||||
config.onAudioSegment?.({
|
||||
audioData: pcmBase64,
|
||||
isLast,
|
||||
});
|
||||
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
},
|
||||
[config]
|
||||
);
|
||||
|
||||
const VOLUME_THRESHOLD = config.volumeThreshold;
|
||||
const SILENCE_DURATION_MS = config.silenceDuration;
|
||||
const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration;
|
||||
const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod;
|
||||
|
||||
// Update segment duration timer
|
||||
useEffect(() => {
|
||||
if (!isDetecting && !isSpeaking) {
|
||||
setSegmentDuration(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const startTime = speechDetectionStartRef.current || Date.now();
|
||||
const interval = setInterval(() => {
|
||||
const elapsed = Date.now() - startTime;
|
||||
setSegmentDuration(elapsed);
|
||||
}, 100);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [isDetecting, isSpeaking]);
|
||||
|
||||
// Listen to microphone data
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
useCallback<MicrophoneDataCallback>(
|
||||
(event) => {
|
||||
if (!isActive || isMuted) return;
|
||||
|
||||
const pcmData: Uint8Array = event.data;
|
||||
|
||||
// Buffer the audio chunk if we're detecting or speaking
|
||||
// Start buffering from first spike to capture beginning of speech
|
||||
if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
|
||||
audioBufferRef.current.push(pcmData);
|
||||
bufferedBytesRef.current += pcmData.length;
|
||||
|
||||
if (
|
||||
speechConfirmedRef.current &&
|
||||
bufferedBytesRef.current >= MIN_CHUNK_BYTES
|
||||
) {
|
||||
flushBufferedAudio(false);
|
||||
}
|
||||
}
|
||||
},
|
||||
[isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
|
||||
)
|
||||
);
|
||||
|
||||
// Listen to volume level for VAD
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
useCallback<VolumeLevelCallback>(
|
||||
(event) => {
|
||||
if (!isActive) return;
|
||||
|
||||
const volumeLevel: number = event.data;
|
||||
setVolume(volumeLevel);
|
||||
|
||||
if (isMuted) return;
|
||||
|
||||
const speechDetected = volumeLevel > VOLUME_THRESHOLD;
|
||||
|
||||
// console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD);
|
||||
|
||||
if (
|
||||
speechDetected &&
|
||||
!isSpeakingRef.current &&
|
||||
!speechConfirmedRef.current
|
||||
) {
|
||||
// Initial speech detection - start tracking
|
||||
if (speechDetectionStartRef.current === null) {
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech started (volume:",
|
||||
volumeLevel.toFixed(4),
|
||||
")"
|
||||
);
|
||||
speechDetectionStartRef.current = Date.now();
|
||||
detectionSilenceStartRef.current = null;
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
setIsDetecting(true);
|
||||
} else {
|
||||
// Volume is back above threshold - reset grace period
|
||||
detectionSilenceStartRef.current = null;
|
||||
|
||||
// Check if speech has been sustained long enough
|
||||
const speechDuration = Date.now() - speechDetectionStartRef.current;
|
||||
if (speechDuration >= SPEECH_CONFIRMATION_MS) {
|
||||
// Speech CONFIRMED
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech confirmed after",
|
||||
speechDuration,
|
||||
"ms"
|
||||
);
|
||||
isSpeakingRef.current = true;
|
||||
speechConfirmedRef.current = true;
|
||||
silenceStartRef.current = null;
|
||||
setIsDetecting(false);
|
||||
setIsSpeaking(true);
|
||||
config.onSpeechStart?.();
|
||||
}
|
||||
}
|
||||
} else if (
|
||||
speechDetected &&
|
||||
isSpeakingRef.current &&
|
||||
speechConfirmedRef.current
|
||||
) {
|
||||
// Continuing confirmed speech
|
||||
silenceStartRef.current = null;
|
||||
} else if (
|
||||
!speechDetected &&
|
||||
!speechConfirmedRef.current &&
|
||||
speechDetectionStartRef.current !== null
|
||||
) {
|
||||
// Volume dropped during detection phase - apply grace period
|
||||
if (detectionSilenceStartRef.current === null) {
|
||||
detectionSilenceStartRef.current = Date.now();
|
||||
} else {
|
||||
const graceDuration = Date.now() - detectionSilenceStartRef.current;
|
||||
if (graceDuration >= DETECTION_GRACE_PERIOD_MS) {
|
||||
// Grace period expired - cancel detection
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech detection cancelled after",
|
||||
graceDuration,
|
||||
"ms grace period"
|
||||
);
|
||||
speechDetectionStartRef.current = null;
|
||||
detectionSilenceStartRef.current = null;
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
setIsDetecting(false);
|
||||
}
|
||||
}
|
||||
} else if (
|
||||
!speechDetected &&
|
||||
isSpeakingRef.current &&
|
||||
speechConfirmedRef.current
|
||||
) {
|
||||
// Potential speech END
|
||||
if (silenceStartRef.current === null) {
|
||||
silenceStartRef.current = Date.now();
|
||||
} else {
|
||||
const silenceDuration = Date.now() - silenceStartRef.current;
|
||||
if (silenceDuration >= SILENCE_DURATION_MS) {
|
||||
// Speech END confirmed
|
||||
console.log(
|
||||
"[SpeechmaticsAudio] Speech ended after",
|
||||
silenceDuration,
|
||||
"ms silence"
|
||||
);
|
||||
isSpeakingRef.current = false;
|
||||
speechConfirmedRef.current = false;
|
||||
speechDetectionStartRef.current = null;
|
||||
silenceStartRef.current = null;
|
||||
setIsSpeaking(false);
|
||||
config.onSpeechEnd?.();
|
||||
|
||||
// Send buffered audio segment
|
||||
flushBufferedAudio(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
[
|
||||
isActive,
|
||||
isMuted,
|
||||
VOLUME_THRESHOLD,
|
||||
SILENCE_DURATION_MS,
|
||||
SPEECH_CONFIRMATION_MS,
|
||||
DETECTION_GRACE_PERIOD_MS,
|
||||
config,
|
||||
flushBufferedAudio,
|
||||
]
|
||||
)
|
||||
);
|
||||
|
||||
const ensureMicrophonePermission = useCallback(async () => {
|
||||
let permissionStatus = microphonePermission;
|
||||
|
||||
if (!permissionStatus?.granted) {
|
||||
try {
|
||||
permissionStatus = await requestMicrophonePermission();
|
||||
} catch (err) {
|
||||
throw new Error("Failed to request microphone permission");
|
||||
}
|
||||
}
|
||||
|
||||
if (!permissionStatus?.granted) {
|
||||
throw new Error(
|
||||
"Microphone permission is required to capture audio. Please enable microphone access in system settings."
|
||||
);
|
||||
}
|
||||
}, [microphonePermission, requestMicrophonePermission]);
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (isActive) {
|
||||
console.log("[SpeechmaticsAudio] Already active");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureMicrophonePermission();
|
||||
|
||||
// Initialize audio if not already initialized
|
||||
if (!audioInitialized) {
|
||||
console.log("[SpeechmaticsAudio] Initializing audio...");
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
console.log("[SpeechmaticsAudio] Audio initialized");
|
||||
}
|
||||
|
||||
console.log("[SpeechmaticsAudio] Starting audio capture...");
|
||||
|
||||
// Start recording
|
||||
toggleRecording(true);
|
||||
|
||||
setIsActive(true);
|
||||
console.log("[SpeechmaticsAudio] Audio capture started successfully");
|
||||
} catch (error) {
|
||||
console.error("[SpeechmaticsAudio] Start error:", error);
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
await stop();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<void> {
|
||||
console.log("[SpeechmaticsAudio] Stopping audio capture...");
|
||||
|
||||
// Stop recording
|
||||
if (isActive) {
|
||||
toggleRecording(false);
|
||||
}
|
||||
|
||||
// Tear down audio session
|
||||
if (audioInitialized) {
|
||||
tearDown();
|
||||
setAudioInitialized(false);
|
||||
console.log("[SpeechmaticsAudio] Audio torn down");
|
||||
}
|
||||
|
||||
// Reset state
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
isSpeakingRef.current = false;
|
||||
speechConfirmedRef.current = false;
|
||||
speechDetectionStartRef.current = null;
|
||||
detectionSilenceStartRef.current = null;
|
||||
silenceStartRef.current = null;
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
setVolume(0);
|
||||
setIsMuted(false);
|
||||
|
||||
console.log("[SpeechmaticsAudio] Audio capture stopped");
|
||||
}
|
||||
|
||||
function toggleMute(): void {
|
||||
setIsMuted((prev) => {
|
||||
const newMuted = !prev;
|
||||
console.log("[SpeechmaticsAudio] Mute toggled:", newMuted);
|
||||
|
||||
if (newMuted) {
|
||||
// Clear any ongoing speech detection/speaking state
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
isSpeakingRef.current = false;
|
||||
speechConfirmedRef.current = false;
|
||||
speechDetectionStartRef.current = null;
|
||||
detectionSilenceStartRef.current = null;
|
||||
silenceStartRef.current = null;
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
}
|
||||
|
||||
return newMuted;
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
toggleMute,
|
||||
isActive,
|
||||
isSpeaking,
|
||||
isDetecting,
|
||||
isMuted,
|
||||
volume,
|
||||
segmentDuration,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user