refactor(app): replace voice hooks with voice runtime and audio engine architecture
This commit is contained in:
Mohamed Boudra
@@ -1 +0,0 @@
|
||||
export * from "./use-audio-player.native";
|
||||
@@ -1,366 +0,0 @@
|
||||
import { useState, useRef } from "react";
|
||||
import {
|
||||
initialize,
|
||||
playPCMData,
|
||||
stopPlayback,
|
||||
pausePlayback,
|
||||
resumePlayback,
|
||||
} from "@boudra/expo-two-way-audio";
|
||||
|
||||
interface QueuedAudio {
|
||||
audioData: Blob;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resample PCM16 audio between sample rates.
|
||||
* Speechmatics expects 16kHz.
|
||||
*/
|
||||
function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array {
|
||||
if (fromRate === toRate) {
|
||||
return pcm;
|
||||
}
|
||||
|
||||
const inputSamples = Math.floor(pcm.length / 2);
|
||||
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
|
||||
const out = new Uint8Array(outputSamples * 2);
|
||||
|
||||
const ratio = fromRate / toRate;
|
||||
|
||||
const readInt16 = (sampleIndex: number): number => {
|
||||
const i = sampleIndex * 2;
|
||||
if (i + 1 >= pcm.length) {
|
||||
return 0;
|
||||
}
|
||||
const lo = pcm[i]!;
|
||||
const hi = pcm[i + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value = value - 0x10000;
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const writeInt16 = (sampleIndex: number, value: number): void => {
|
||||
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
|
||||
const i = sampleIndex * 2;
|
||||
out[i] = clamped & 0xff;
|
||||
out[i + 1] = (clamped >> 8) & 0xff;
|
||||
};
|
||||
|
||||
for (let i = 0; i < outputSamples; i++) {
|
||||
const srcPos = i * ratio;
|
||||
const i0 = Math.floor(srcPos);
|
||||
const frac = srcPos - i0;
|
||||
const s0 = readInt16(i0);
|
||||
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
|
||||
writeInt16(i, s0 + (s1 - s0) * frac);
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function parsePcmSampleRate(mimeType: string): number | null {
|
||||
const match = /rate=(\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
}
|
||||
|
||||
export interface AudioPlayerOptions {
|
||||
isDetecting?: () => boolean;
|
||||
isSpeaking?: () => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio playback using Speechmatics two-way audio with echo cancellation
|
||||
*/
|
||||
export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
const queueRef = useRef<QueuedAudio[]>([]);
|
||||
const suppressedQueueRef = useRef<QueuedAudio[]>([]);
|
||||
const isProcessingQueueRef = useRef(false);
|
||||
const activePlaybackRef = useRef<{
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
} | null>(null);
|
||||
const playbackTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
async function play(audioData: Blob): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Check if we should suppress playback due to voice detection/speaking
|
||||
const shouldSuppress =
|
||||
(options?.isDetecting && options.isDetecting()) ||
|
||||
(options?.isSpeaking && options.isSpeaking());
|
||||
|
||||
if (shouldSuppress) {
|
||||
console.log("[AudioPlayer] Suppressing playback - voice detection/speaking active");
|
||||
// Add to suppressed queue instead
|
||||
suppressedQueueRef.current.push({ audioData, resolve, reject });
|
||||
|
||||
// Start checking for when flags clear
|
||||
startCheckingForClearFlags();
|
||||
return;
|
||||
}
|
||||
|
||||
// Add to queue with its promise handlers
|
||||
queueRef.current.push({ audioData, resolve, reject });
|
||||
|
||||
// Start processing queue if not already processing
|
||||
if (!isProcessingQueueRef.current) {
|
||||
processQueue();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function startCheckingForClearFlags(): void {
|
||||
// Already checking
|
||||
if (checkIntervalRef.current !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("[AudioPlayer] Starting to check for clear flags");
|
||||
|
||||
checkIntervalRef.current = setInterval(() => {
|
||||
const isStillBlocked =
|
||||
(options?.isDetecting && options.isDetecting()) ||
|
||||
(options?.isSpeaking && options.isSpeaking());
|
||||
|
||||
if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
|
||||
console.log("[AudioPlayer] Flags cleared - moving suppressed queue to main queue");
|
||||
|
||||
// Move all suppressed items to main queue
|
||||
const suppressedItems = [...suppressedQueueRef.current];
|
||||
suppressedQueueRef.current = [];
|
||||
|
||||
// Add to front of main queue (they were waiting)
|
||||
queueRef.current = [...suppressedItems, ...queueRef.current];
|
||||
|
||||
// Stop checking
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
|
||||
// Start processing if not already
|
||||
if (!isProcessingQueueRef.current) {
|
||||
processQueue();
|
||||
}
|
||||
} else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
|
||||
// No more suppressed items and flags are clear - stop checking
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
}
|
||||
}, 100); // Check every 100ms
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = true;
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
// Before processing each item, check if flags became active
|
||||
const shouldSuppress =
|
||||
(options?.isDetecting && options.isDetecting()) ||
|
||||
(options?.isSpeaking && options.isSpeaking());
|
||||
|
||||
if (shouldSuppress) {
|
||||
console.log("[AudioPlayer] Flags became active during processing - moving remaining queue to suppressed");
|
||||
// Move remaining queue to suppressed
|
||||
suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
|
||||
queueRef.current = [];
|
||||
startCheckingForClearFlags();
|
||||
break;
|
||||
}
|
||||
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
async function processNextInQueue(): Promise<void> {
|
||||
if (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
} else {
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audioData: Blob): Promise<number> {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
activePlaybackRef.current = { resolve, reject };
|
||||
try {
|
||||
console.log(
|
||||
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
|
||||
);
|
||||
|
||||
// Initialize audio if not already initialized
|
||||
if (!audioInitialized) {
|
||||
console.log("[AudioPlayer] Initializing audio...");
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
console.log(
|
||||
"[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)"
|
||||
);
|
||||
}
|
||||
|
||||
// Workaround: Resume playback before playing new audio to ensure the audio engine is ready
|
||||
// This fixes the issue where playback doesn't work after calling stopPlayback()
|
||||
console.log("[AudioPlayer] Resuming playback engine...");
|
||||
resumePlayback();
|
||||
|
||||
// Get PCM data from blob (server sends PCM16)
|
||||
const arrayBuffer = await audioData.arrayBuffer();
|
||||
const pcm = new Uint8Array(arrayBuffer);
|
||||
|
||||
const inputRate = parsePcmSampleRate(audioData.type || "") ?? 24000;
|
||||
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
|
||||
|
||||
// Calculate total duration
|
||||
const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample
|
||||
const durationSec = samples / 16000; // 16kHz sample rate
|
||||
|
||||
const audioSizeKb = (pcm16k.length / 1024).toFixed(2);
|
||||
console.log(
|
||||
"[AudioPlayer] 🔊 Playing audio:",
|
||||
audioSizeKb,
|
||||
"KB, duration:",
|
||||
durationSec.toFixed(2),
|
||||
"s"
|
||||
);
|
||||
|
||||
setIsPlaying(true);
|
||||
|
||||
// Play entire PCM data at once through Speechmatics
|
||||
playPCMData(pcm16k);
|
||||
|
||||
// Clear any existing timeout
|
||||
if (playbackTimeoutRef.current) {
|
||||
clearTimeout(playbackTimeoutRef.current);
|
||||
}
|
||||
|
||||
// Wait for playback to finish (estimate based on duration)
|
||||
playbackTimeoutRef.current = setTimeout(() => {
|
||||
console.log("[AudioPlayer] ✅ Playback finished");
|
||||
setIsPlaying(false);
|
||||
playbackTimeoutRef.current = null;
|
||||
activePlaybackRef.current = null;
|
||||
resolve(durationSec);
|
||||
}, durationSec * 1000);
|
||||
} catch (error) {
|
||||
console.error("[AudioPlayer] Error playing audio:", error);
|
||||
|
||||
// Clear timeout on error
|
||||
if (playbackTimeoutRef.current) {
|
||||
clearTimeout(playbackTimeoutRef.current);
|
||||
playbackTimeoutRef.current = null;
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
activePlaybackRef.current = null;
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (isPlaying) {
|
||||
console.log("[AudioPlayer] 🛑 Stopping playback (interrupted)");
|
||||
|
||||
// Stop native playback
|
||||
stopPlayback();
|
||||
|
||||
// Clear playback timeout
|
||||
if (playbackTimeoutRef.current) {
|
||||
clearTimeout(playbackTimeoutRef.current);
|
||||
playbackTimeoutRef.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
|
||||
// Reject the currently playing promise, if any.
|
||||
if (activePlaybackRef.current) {
|
||||
activePlaybackRef.current.reject(new Error("Playback stopped"));
|
||||
activePlaybackRef.current = null;
|
||||
}
|
||||
|
||||
// Reject all pending promises in the main queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error("Playback stopped"));
|
||||
}
|
||||
|
||||
// Reject all pending promises in the suppressed queue
|
||||
while (suppressedQueueRef.current.length > 0) {
|
||||
const item = suppressedQueueRef.current.shift()!;
|
||||
item.reject(new Error("Playback stopped"));
|
||||
}
|
||||
|
||||
// Clear check interval
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
function clearQueue(): void {
|
||||
// Reject all pending promises in the main queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error("Queue cleared"));
|
||||
}
|
||||
|
||||
// Reject all pending promises in the suppressed queue
|
||||
while (suppressedQueueRef.current.length > 0) {
|
||||
const item = suppressedQueueRef.current.shift()!;
|
||||
item.reject(new Error("Queue cleared"));
|
||||
}
|
||||
|
||||
// Clear check interval
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
play,
|
||||
stop,
|
||||
isPlaying: () => isPlaying,
|
||||
clearQueue,
|
||||
warmup: async () => {
|
||||
if (!audioInitialized) {
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
}
|
||||
// Ensure playback engine isn't suspended after a previous stop.
|
||||
resumePlayback();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,275 +0,0 @@
|
||||
import { useMemo, useRef, useState } from "react";
|
||||
|
||||
export interface AudioPlayerOptions {
|
||||
isDetecting?: () => boolean;
|
||||
isSpeaking?: () => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Web audio player for server-provided audio chunks.
|
||||
*
|
||||
* Supports:
|
||||
* - `audio/pcm` (assumed PCM16 LE, mono, default 24kHz unless `rate=` is present)
|
||||
* - formats supported by `AudioContext.decodeAudioData` (e.g. mp3)
|
||||
*/
|
||||
export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
const [isPlayingState, setIsPlayingState] = useState(false);
|
||||
|
||||
type QueuedAudio = {
|
||||
audioData: Blob;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
};
|
||||
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const queueRef = useRef<QueuedAudio[]>([]);
|
||||
const suppressedQueueRef = useRef<QueuedAudio[]>([]);
|
||||
const isProcessingQueueRef = useRef(false);
|
||||
const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const activePlaybackRef = useRef<{
|
||||
source: AudioBufferSourceNode;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
settled: boolean;
|
||||
} | null>(null);
|
||||
|
||||
const decodeAudioData = async (context: AudioContext, buffer: ArrayBuffer): Promise<AudioBuffer> => {
|
||||
const maybePromise = context.decodeAudioData(buffer);
|
||||
if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
|
||||
return maybePromise as Promise<AudioBuffer>;
|
||||
}
|
||||
return await new Promise<AudioBuffer>((resolve, reject) => {
|
||||
context.decodeAudioData(buffer, resolve, reject);
|
||||
});
|
||||
};
|
||||
|
||||
const parsePcmSampleRate = (mimeType: string): number | null => {
|
||||
const match = /rate=(\\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
};
|
||||
|
||||
const pcm16LeToAudioBuffer = (context: AudioContext, bytes: Uint8Array, sampleRate: number): AudioBuffer => {
|
||||
const sampleCount = Math.floor(bytes.length / 2);
|
||||
const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
|
||||
const channel = audioBuffer.getChannelData(0);
|
||||
for (let i = 0; i < sampleCount; i++) {
|
||||
const lo = bytes[i * 2]!;
|
||||
const hi = bytes[i * 2 + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value = value - 0x10000;
|
||||
}
|
||||
channel[i] = value / 0x8000;
|
||||
}
|
||||
return audioBuffer;
|
||||
};
|
||||
|
||||
const ensureContext = async (): Promise<AudioContext> => {
|
||||
if (audioContextRef.current) {
|
||||
if (audioContextRef.current.state === "suspended") {
|
||||
try {
|
||||
await audioContextRef.current.resume();
|
||||
} catch {
|
||||
// Best effort. If this fails due to autoplay policies, a later user gesture
|
||||
// (or explicit warmup call) can unlock it.
|
||||
}
|
||||
}
|
||||
return audioContextRef.current;
|
||||
}
|
||||
const context = new AudioContext();
|
||||
if (context.state === "suspended") {
|
||||
try {
|
||||
await context.resume();
|
||||
} catch {
|
||||
// See note above.
|
||||
}
|
||||
}
|
||||
audioContextRef.current = context;
|
||||
return context;
|
||||
};
|
||||
|
||||
const shouldSuppressPlayback = (): boolean => {
|
||||
return Boolean(options?.isDetecting?.()) || Boolean(options?.isSpeaking?.());
|
||||
};
|
||||
|
||||
const startCheckingForClearFlags = (): void => {
|
||||
if (checkIntervalRef.current !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
checkIntervalRef.current = setInterval(() => {
|
||||
const isStillBlocked = shouldSuppressPlayback();
|
||||
if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
|
||||
const suppressedItems = [...suppressedQueueRef.current];
|
||||
suppressedQueueRef.current = [];
|
||||
queueRef.current = [...suppressedItems, ...queueRef.current];
|
||||
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
|
||||
if (!isProcessingQueueRef.current) {
|
||||
void processQueue();
|
||||
}
|
||||
} else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
}
|
||||
}, 100);
|
||||
};
|
||||
|
||||
const playAudio = async (audioData: Blob): Promise<number> => {
|
||||
const context = await ensureContext();
|
||||
const arrayBuffer = await audioData.arrayBuffer();
|
||||
|
||||
let audioBuffer: AudioBuffer;
|
||||
const type = (audioData.type || "").toLowerCase();
|
||||
|
||||
if (type.startsWith("audio/pcm")) {
|
||||
const sampleRate = parsePcmSampleRate(type) ?? 24000;
|
||||
audioBuffer = pcm16LeToAudioBuffer(context, new Uint8Array(arrayBuffer), sampleRate);
|
||||
} else {
|
||||
audioBuffer = await decodeAudioData(context, arrayBuffer);
|
||||
}
|
||||
|
||||
const durationSec = audioBuffer.duration;
|
||||
const source = context.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(context.destination);
|
||||
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
activePlaybackRef.current = { source, resolve, reject, settled: false };
|
||||
setIsPlayingState(true);
|
||||
|
||||
const settleOnce = (fn: () => void) => {
|
||||
const active = activePlaybackRef.current;
|
||||
if (!active || active.source !== source || active.settled) {
|
||||
return;
|
||||
}
|
||||
active.settled = true;
|
||||
activePlaybackRef.current = null;
|
||||
setIsPlayingState(false);
|
||||
fn();
|
||||
};
|
||||
|
||||
source.onended = () => {
|
||||
settleOnce(() => resolve(durationSec));
|
||||
};
|
||||
|
||||
try {
|
||||
source.start();
|
||||
} catch (e) {
|
||||
settleOnce(() => reject(e instanceof Error ? e : new Error(String(e))));
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const processQueue = async (): Promise<void> => {
|
||||
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = true;
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
if (shouldSuppressPlayback()) {
|
||||
suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
|
||||
queueRef.current = [];
|
||||
startCheckingForClearFlags();
|
||||
break;
|
||||
}
|
||||
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
};
|
||||
|
||||
const play = async (audioData: Blob): Promise<number> => {
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
if (shouldSuppressPlayback()) {
|
||||
suppressedQueueRef.current.push({ audioData, resolve, reject });
|
||||
startCheckingForClearFlags();
|
||||
return;
|
||||
}
|
||||
|
||||
queueRef.current.push({ audioData, resolve, reject });
|
||||
if (!isProcessingQueueRef.current) {
|
||||
void processQueue();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const stop = (): void => {
|
||||
// Stop currently playing audio (and reject its promise)
|
||||
if (activePlaybackRef.current) {
|
||||
const active = activePlaybackRef.current;
|
||||
activePlaybackRef.current = null;
|
||||
try {
|
||||
active.source.onended = null;
|
||||
active.source.stop();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (!active.settled) {
|
||||
active.settled = true;
|
||||
active.reject(new Error("Playback stopped"));
|
||||
}
|
||||
}
|
||||
|
||||
setIsPlayingState(false);
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
queueRef.current.shift()!.reject(new Error("Playback stopped"));
|
||||
}
|
||||
while (suppressedQueueRef.current.length > 0) {
|
||||
suppressedQueueRef.current.shift()!.reject(new Error("Playback stopped"));
|
||||
}
|
||||
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
};
|
||||
|
||||
const clearQueue = (): void => {
|
||||
while (queueRef.current.length > 0) {
|
||||
queueRef.current.shift()!.reject(new Error("Queue cleared"));
|
||||
}
|
||||
while (suppressedQueueRef.current.length > 0) {
|
||||
suppressedQueueRef.current.shift()!.reject(new Error("Queue cleared"));
|
||||
}
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
};
|
||||
|
||||
return useMemo(
|
||||
() => ({
|
||||
play,
|
||||
stop,
|
||||
isPlaying: () => isPlayingState,
|
||||
clearQueue,
|
||||
warmup: async () => {
|
||||
await ensureContext();
|
||||
},
|
||||
}),
|
||||
[isPlayingState]
|
||||
);
|
||||
}
|
||||
@@ -1,44 +1,124 @@
|
||||
import { useCallback, useEffect, useRef } from "react";
|
||||
import { Buffer } from "buffer";
|
||||
import { useState } from "react";
|
||||
|
||||
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
|
||||
import { createAudioEngine } from "@/voice/audio-engine";
|
||||
|
||||
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
|
||||
|
||||
let nextDictationAudioSourceInstanceId = 1;
|
||||
|
||||
function getDictationTraceStack(): string | undefined {
|
||||
const stack = new Error().stack;
|
||||
if (!stack) {
|
||||
return undefined;
|
||||
}
|
||||
return stack
|
||||
.split("\n")
|
||||
.slice(2, 7)
|
||||
.map((line) => line.trim())
|
||||
.join(" | ");
|
||||
}
|
||||
|
||||
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||
const onPcmSegmentRef = useRef(config.onPcmSegment);
|
||||
const onErrorRef = useRef(config.onError);
|
||||
const [volume, setVolume] = useState(0);
|
||||
const sourceIdRef = useRef<number | null>(null);
|
||||
const engineRef = useRef<ReturnType<typeof createAudioEngine> | null>(null);
|
||||
|
||||
if (sourceIdRef.current === null) {
|
||||
sourceIdRef.current = nextDictationAudioSourceInstanceId++;
|
||||
console.log("[DictationAudioSource] instance_created", {
|
||||
sourceId: sourceIdRef.current,
|
||||
stack: getDictationTraceStack(),
|
||||
});
|
||||
}
|
||||
|
||||
const sourceId = sourceIdRef.current;
|
||||
|
||||
console.log("[DictationAudioSource] render", {
|
||||
sourceId,
|
||||
hasEngine: Boolean(engineRef.current),
|
||||
});
|
||||
|
||||
const getOrCreateEngine = useCallback(() => {
|
||||
if (engineRef.current) {
|
||||
return engineRef.current;
|
||||
}
|
||||
|
||||
console.log("[DictationAudioSource] create_engine", {
|
||||
sourceId,
|
||||
stack: getDictationTraceStack(),
|
||||
});
|
||||
engineRef.current = createAudioEngine(
|
||||
{
|
||||
onCaptureData: (pcm) => {
|
||||
onPcmSegmentRef.current(Buffer.from(pcm).toString("base64"));
|
||||
},
|
||||
onVolumeLevel: (level) => {
|
||||
setVolume(level);
|
||||
},
|
||||
onError: (error) => {
|
||||
onErrorRef.current?.(error);
|
||||
},
|
||||
},
|
||||
{
|
||||
traceLabel: `dictation:${sourceId}`,
|
||||
}
|
||||
);
|
||||
return engineRef.current;
|
||||
}, [sourceId]);
|
||||
|
||||
useEffect(() => {
|
||||
onPcmSegmentRef.current = config.onPcmSegment;
|
||||
onErrorRef.current = config.onError;
|
||||
}, [config.onPcmSegment, config.onError]);
|
||||
|
||||
const speechmatics = useSpeechmaticsAudio({
|
||||
enableContinuousStreaming: true,
|
||||
onAudioSegment: ({ audioData }) => {
|
||||
onPcmSegmentRef.current(audioData);
|
||||
},
|
||||
onError: (err) => {
|
||||
onErrorRef.current?.(err);
|
||||
},
|
||||
volumeThreshold: 0.3,
|
||||
silenceDuration: 2000,
|
||||
speechConfirmationDuration: 300,
|
||||
detectionGracePeriod: 200,
|
||||
});
|
||||
useEffect(() => {
|
||||
console.log("[DictationAudioSource] mount", {
|
||||
sourceId,
|
||||
});
|
||||
return () => {
|
||||
console.log("[DictationAudioSource] unmount", {
|
||||
sourceId,
|
||||
});
|
||||
};
|
||||
}, [sourceId]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
await speechmatics.start();
|
||||
}, [speechmatics]);
|
||||
console.log("[DictationAudioSource] start", {
|
||||
sourceId,
|
||||
});
|
||||
const engine = getOrCreateEngine();
|
||||
await engine.initialize();
|
||||
await engine.startCapture();
|
||||
}, [getOrCreateEngine, sourceId]);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
await speechmatics.stop();
|
||||
}, [speechmatics]);
|
||||
console.log("[DictationAudioSource] stop", {
|
||||
sourceId,
|
||||
hasEngine: Boolean(engineRef.current),
|
||||
});
|
||||
await engineRef.current?.stopCapture();
|
||||
setVolume(0);
|
||||
}, [sourceId]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
const engine = engineRef.current;
|
||||
engineRef.current = null;
|
||||
console.log("[DictationAudioSource] destroy_engine", {
|
||||
sourceId,
|
||||
hadEngine: Boolean(engine),
|
||||
});
|
||||
void engine?.destroy().catch(() => undefined);
|
||||
};
|
||||
}, [sourceId]);
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
volume: speechmatics.volume,
|
||||
volume,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { parsePcm16Wav } from "@/utils/pcm16-wav";
|
||||
import { getTauri } from "@/utils/tauri";
|
||||
|
||||
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
|
||||
@@ -65,84 +66,6 @@ const int16ToFloat32 = (input: Int16Array): Float32Array => {
|
||||
return out;
|
||||
};
|
||||
|
||||
type Pcm16Wav = {
|
||||
sampleRate: number;
|
||||
samples: Int16Array;
|
||||
};
|
||||
|
||||
const parsePcm16Wav = (buffer: ArrayBuffer): Pcm16Wav | null => {
|
||||
if (buffer.byteLength < 44) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const view = new DataView(buffer);
|
||||
const readAscii = (offset: number, length: number): string => {
|
||||
let out = "";
|
||||
for (let i = 0; i < length; i += 1) {
|
||||
out += String.fromCharCode(view.getUint8(offset + i));
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
if (readAscii(0, 4) !== "RIFF" || readAscii(8, 4) !== "WAVE") {
|
||||
return null;
|
||||
}
|
||||
|
||||
let offset = 12;
|
||||
let channels = 0;
|
||||
let sampleRate = 0;
|
||||
let bitsPerSample = 0;
|
||||
let dataOffset = 0;
|
||||
let dataSize = 0;
|
||||
|
||||
while (offset + 8 <= buffer.byteLength) {
|
||||
const chunkId = readAscii(offset, 4);
|
||||
const chunkSize = view.getUint32(offset + 4, true);
|
||||
const chunkDataOffset = offset + 8;
|
||||
if (chunkDataOffset + chunkSize > buffer.byteLength) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (chunkId === "fmt " && chunkSize >= 16) {
|
||||
const audioFormat = view.getUint16(chunkDataOffset, true);
|
||||
channels = view.getUint16(chunkDataOffset + 2, true);
|
||||
sampleRate = view.getUint32(chunkDataOffset + 4, true);
|
||||
bitsPerSample = view.getUint16(chunkDataOffset + 14, true);
|
||||
if (audioFormat !== 1) {
|
||||
return null;
|
||||
}
|
||||
} else if (chunkId === "data") {
|
||||
dataOffset = chunkDataOffset;
|
||||
dataSize = chunkSize;
|
||||
break;
|
||||
}
|
||||
|
||||
offset = chunkDataOffset + chunkSize + (chunkSize % 2);
|
||||
}
|
||||
|
||||
if (!dataOffset || !dataSize || sampleRate <= 0 || bitsPerSample !== 16 || channels <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sampleCount = Math.floor(dataSize / 2);
|
||||
const interleaved = new Int16Array(buffer, dataOffset, sampleCount);
|
||||
|
||||
if (channels === 1) {
|
||||
return { sampleRate, samples: new Int16Array(interleaved) };
|
||||
}
|
||||
|
||||
const frameCount = Math.floor(interleaved.length / channels);
|
||||
const mono = new Int16Array(frameCount);
|
||||
for (let frame = 0; frame < frameCount; frame += 1) {
|
||||
let sum = 0;
|
||||
for (let ch = 0; ch < channels; ch += 1) {
|
||||
sum += interleaved[frame * channels + ch] ?? 0;
|
||||
}
|
||||
mono[frame] = Math.round(sum / channels);
|
||||
}
|
||||
return { sampleRate, samples: mono };
|
||||
};
|
||||
|
||||
const int16ToBase64 = (pcm: Int16Array): string => {
|
||||
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
||||
let binary = "";
|
||||
|
||||
@@ -267,8 +267,6 @@ export function useSidebarWorkspacesList(options?: {
|
||||
const value = options?.serverId
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null
|
||||
}, [options?.serverId])
|
||||
const enabled = options?.enabled ?? true
|
||||
|
||||
const persistedProjectOrder = useSidebarOrderStore((state) =>
|
||||
serverId ? (state.projectOrderByServerId[serverId] ?? EMPTY_ORDER) : EMPTY_ORDER
|
||||
)
|
||||
@@ -304,6 +302,13 @@ export function useSidebarWorkspacesList(options?: {
|
||||
)
|
||||
|
||||
const projects = useMemo(() => {
|
||||
console.log('[useSidebarWorkspacesList] build_projects', {
|
||||
serverId,
|
||||
hasSessionWorkspaces: Boolean(sessionWorkspaces),
|
||||
workspaceCount: sessionWorkspaces?.size ?? 0,
|
||||
projectOrderLength: persistedProjectOrder.length,
|
||||
workspaceOrderScopeCount: Object.keys(persistedWorkspaceOrderByScope).length,
|
||||
})
|
||||
if (!sessionWorkspaces || sessionWorkspaces.size === 0 || !serverId) {
|
||||
return EMPTY_PROJECTS
|
||||
}
|
||||
@@ -315,6 +320,36 @@ export function useSidebarWorkspacesList(options?: {
|
||||
})
|
||||
}, [persistedProjectOrder, persistedWorkspaceOrderByScope, serverId, sessionWorkspaces])
|
||||
|
||||
useEffect(() => {
|
||||
console.log('[useSidebarWorkspacesList] inputs_changed', {
|
||||
serverId,
|
||||
connectionStatus,
|
||||
hasHydratedWorkspaces,
|
||||
workspaceCount: sessionWorkspaces?.size ?? 0,
|
||||
projectOrderLength: persistedProjectOrder.length,
|
||||
workspaceOrderScopeCount: Object.keys(persistedWorkspaceOrderByScope).length,
|
||||
})
|
||||
}, [
|
||||
connectionStatus,
|
||||
hasHydratedWorkspaces,
|
||||
persistedProjectOrder,
|
||||
persistedWorkspaceOrderByScope,
|
||||
serverId,
|
||||
sessionWorkspaces,
|
||||
])
|
||||
|
||||
useEffect(() => {
|
||||
console.log('[useSidebarWorkspacesList] projects_changed', {
|
||||
serverId,
|
||||
projectCount: projects.length,
|
||||
projectKeys: projects.map((project) => project.projectKey),
|
||||
workspaceCounts: projects.map((project) => ({
|
||||
projectKey: project.projectKey,
|
||||
workspaceCount: project.workspaces.length,
|
||||
})),
|
||||
})
|
||||
}, [projects, serverId])
|
||||
|
||||
useEffect(() => {
|
||||
if (!serverId || projects.length === 0) {
|
||||
return
|
||||
@@ -345,7 +380,7 @@ export function useSidebarWorkspacesList(options?: {
|
||||
}, [persistedProjectOrder, persistedWorkspaceOrderByScope, projects, serverId])
|
||||
|
||||
const refreshAll = useCallback(() => {
|
||||
if (!isActive || !serverId || connectionStatus !== 'online' || !enabled) {
|
||||
if (!isActive || !serverId || connectionStatus !== 'online') {
|
||||
return
|
||||
}
|
||||
const client = runtime.getClient(serverId)
|
||||
@@ -377,7 +412,7 @@ export function useSidebarWorkspacesList(options?: {
|
||||
// ignore explicit refresh failures; hook keeps existing data
|
||||
}
|
||||
})()
|
||||
}, [connectionStatus, enabled, isActive, runtime, serverId])
|
||||
}, [connectionStatus, isActive, runtime, serverId])
|
||||
|
||||
const isLoading = isActive && Boolean(serverId) && connectionStatus === 'online' && !hasHydratedWorkspaces
|
||||
const isInitialLoad = isLoading && projects.length === 0
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
export * from "./use-speechmatics-audio.native";
|
||||
@@ -1,274 +0,0 @@
|
||||
import { useState, useEffect, useRef, useCallback } from "react";
|
||||
import {
|
||||
initialize,
|
||||
useMicrophonePermissions,
|
||||
toggleRecording,
|
||||
tearDown,
|
||||
useExpoTwoWayAudioEventListener,
|
||||
type MicrophoneDataCallback,
|
||||
type VolumeLevelCallback,
|
||||
} from "@boudra/expo-two-way-audio";
|
||||
|
||||
import { SpeechSegmenter } from "@/voice/speech-segmenter";
|
||||
|
||||
export interface SpeechmaticsAudioConfig {
|
||||
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
/** When true, stream microphone PCM continuously without VAD gating. */
|
||||
enableContinuousStreaming?: boolean;
|
||||
volumeThreshold: number; // Volume threshold for speech detection (0-1)
|
||||
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
|
||||
confirmedDropGracePeriod?: number;
|
||||
silenceDuration: number; // ms of silence before ending segment
|
||||
speechConfirmationDuration: number; // ms of sustained speech before confirming
|
||||
detectionGracePeriod: number; // ms grace period for volume dips during detection
|
||||
}
|
||||
|
||||
export interface SpeechmaticsAudio {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
toggleMute: () => void;
|
||||
isActive: boolean;
|
||||
isSpeaking: boolean;
|
||||
isDetecting: boolean;
|
||||
isMuted: boolean;
|
||||
volume: number;
|
||||
segmentDuration: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio
|
||||
*/
|
||||
export function useSpeechmaticsAudio(
|
||||
config: SpeechmaticsAudioConfig
|
||||
): SpeechmaticsAudio {
|
||||
const [microphonePermission, requestMicrophonePermission] =
|
||||
useMicrophonePermissions();
|
||||
const [isActive, setIsActive] = useState(false);
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
const [isDetecting, setIsDetecting] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
const [volume, setVolume] = useState(0);
|
||||
const [isMuted, setIsMuted] = useState(false);
|
||||
const [segmentDuration, setSegmentDuration] = useState(0);
|
||||
|
||||
const enableContinuousStreaming = config.enableContinuousStreaming === true;
|
||||
|
||||
const isActiveRef = useRef(isActive);
|
||||
useEffect(() => {
|
||||
isActiveRef.current = isActive;
|
||||
}, [isActive]);
|
||||
|
||||
const isMutedRef = useRef(isMuted);
|
||||
useEffect(() => {
|
||||
isMutedRef.current = isMuted;
|
||||
}, [isMuted]);
|
||||
|
||||
const callbacksRef = useRef({
|
||||
onAudioSegment: config.onAudioSegment,
|
||||
onSpeechStart: config.onSpeechStart,
|
||||
onSpeechEnd: config.onSpeechEnd,
|
||||
});
|
||||
useEffect(() => {
|
||||
callbacksRef.current = {
|
||||
onAudioSegment: config.onAudioSegment,
|
||||
onSpeechStart: config.onSpeechStart,
|
||||
onSpeechEnd: config.onSpeechEnd,
|
||||
};
|
||||
}, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd]);
|
||||
|
||||
const segmenterRef = useRef<SpeechSegmenter | null>(null);
|
||||
if (segmenterRef.current === null) {
|
||||
segmenterRef.current = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming,
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
|
||||
silenceDurationMs: config.silenceDuration,
|
||||
speechConfirmationMs: config.speechConfirmationDuration,
|
||||
detectionGracePeriodMs: config.detectionGracePeriod,
|
||||
},
|
||||
{
|
||||
onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
|
||||
onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
|
||||
onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
|
||||
onDetectingChange: (next) => setIsDetecting(next),
|
||||
onSpeakingChange: (next) => setIsSpeaking(next),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
segmenterRef.current?.updateConfig({
|
||||
enableContinuousStreaming,
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
|
||||
silenceDurationMs: config.silenceDuration,
|
||||
speechConfirmationMs: config.speechConfirmationDuration,
|
||||
detectionGracePeriodMs: config.detectionGracePeriod,
|
||||
});
|
||||
}, [
|
||||
enableContinuousStreaming,
|
||||
config.volumeThreshold,
|
||||
config.confirmedDropGracePeriod,
|
||||
config.silenceDuration,
|
||||
config.speechConfirmationDuration,
|
||||
config.detectionGracePeriod,
|
||||
]);
|
||||
|
||||
// Update segment duration timer
|
||||
useEffect(() => {
|
||||
if (!isDetecting && !isSpeaking) {
|
||||
setSegmentDuration(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
|
||||
const interval = setInterval(() => {
|
||||
const elapsed = Date.now() - startTime;
|
||||
setSegmentDuration(elapsed);
|
||||
}, 100);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [isDetecting, isSpeaking]);
|
||||
|
||||
// Listen to microphone data
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
useCallback<MicrophoneDataCallback>(
|
||||
(event) => {
|
||||
if (!isActiveRef.current || isMutedRef.current) return;
|
||||
|
||||
const pcmData: Uint8Array = event.data;
|
||||
segmenterRef.current?.pushPcmChunk(pcmData);
|
||||
},
|
||||
[]
|
||||
)
|
||||
);
|
||||
|
||||
// Listen to volume level for VAD
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
useCallback<VolumeLevelCallback>(
|
||||
(event) => {
|
||||
if (!isActiveRef.current) return;
|
||||
|
||||
const volumeLevel: number = event.data;
|
||||
setVolume(volumeLevel);
|
||||
|
||||
if (isMutedRef.current) return;
|
||||
segmenterRef.current?.pushVolumeLevel(volumeLevel, Date.now());
|
||||
},
|
||||
[]
|
||||
)
|
||||
);
|
||||
|
||||
const ensureMicrophonePermission = useCallback(async () => {
|
||||
let permissionStatus = microphonePermission;
|
||||
|
||||
if (!permissionStatus?.granted) {
|
||||
try {
|
||||
permissionStatus = await requestMicrophonePermission();
|
||||
} catch (err) {
|
||||
throw new Error("Failed to request microphone permission");
|
||||
}
|
||||
}
|
||||
|
||||
if (!permissionStatus?.granted) {
|
||||
throw new Error(
|
||||
"Microphone permission is required to capture audio. Please enable microphone access in system settings."
|
||||
);
|
||||
}
|
||||
}, [microphonePermission, requestMicrophonePermission]);
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (isActive) {
|
||||
console.log("[SpeechmaticsAudio] Already active");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureMicrophonePermission();
|
||||
|
||||
// Initialize audio if not already initialized
|
||||
if (!audioInitialized) {
|
||||
console.log("[SpeechmaticsAudio] Initializing audio...");
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
console.log("[SpeechmaticsAudio] Audio initialized");
|
||||
}
|
||||
|
||||
console.log("[SpeechmaticsAudio] Starting audio capture...");
|
||||
|
||||
// Start recording
|
||||
toggleRecording(true);
|
||||
|
||||
setIsActive(true);
|
||||
console.log("[SpeechmaticsAudio] Audio capture started successfully");
|
||||
} catch (error) {
|
||||
console.error("[SpeechmaticsAudio] Start error:", error);
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
await stop();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<void> {
|
||||
console.log("[SpeechmaticsAudio] Stopping audio capture...");
|
||||
|
||||
// Stop recording
|
||||
if (isActive) {
|
||||
toggleRecording(false);
|
||||
}
|
||||
|
||||
segmenterRef.current?.stop(Date.now());
|
||||
|
||||
// Tear down audio session
|
||||
if (audioInitialized) {
|
||||
tearDown();
|
||||
setAudioInitialized(false);
|
||||
console.log("[SpeechmaticsAudio] Audio torn down");
|
||||
}
|
||||
|
||||
// Reset state
|
||||
segmenterRef.current?.reset();
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
setVolume(0);
|
||||
setIsMuted(false);
|
||||
|
||||
console.log("[SpeechmaticsAudio] Audio capture stopped");
|
||||
}
|
||||
|
||||
function toggleMute(): void {
|
||||
setIsMuted((prev) => {
|
||||
const newMuted = !prev;
|
||||
console.log("[SpeechmaticsAudio] Mute toggled:", newMuted);
|
||||
|
||||
if (newMuted) {
|
||||
// Clear any ongoing speech detection/speaking state
|
||||
segmenterRef.current?.reset();
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
}
|
||||
|
||||
return newMuted;
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
toggleMute,
|
||||
isActive,
|
||||
isSpeaking,
|
||||
isDetecting,
|
||||
isMuted,
|
||||
volume,
|
||||
segmentDuration,
|
||||
};
|
||||
}
|
||||
@@ -1,410 +0,0 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { SpeechSegmenter } from "@/voice/speech-segmenter";
|
||||
import { getTauri } from "@/utils/tauri";
|
||||
|
||||
export interface SpeechmaticsAudioConfig {
|
||||
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
/** When true, stream microphone PCM continuously without VAD gating. */
|
||||
enableContinuousStreaming?: boolean;
|
||||
volumeThreshold: number; // 0-1
|
||||
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
|
||||
confirmedDropGracePeriod?: number;
|
||||
silenceDuration: number; // ms of silence before ending segment
|
||||
speechConfirmationDuration: number; // ms of sustained speech before confirming
|
||||
detectionGracePeriod: number; // ms grace period for volume dips during detection
|
||||
}
|
||||
|
||||
export interface SpeechmaticsAudio {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
toggleMute: () => void;
|
||||
isActive: boolean;
|
||||
isSpeaking: boolean;
|
||||
isDetecting: boolean;
|
||||
isMuted: boolean;
|
||||
volume: number;
|
||||
segmentDuration: number;
|
||||
}
|
||||
|
||||
const getAudioContextCtor = (): (typeof AudioContext) | null => {
|
||||
if (typeof window === "undefined") {
|
||||
return null;
|
||||
}
|
||||
const ctor =
|
||||
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext ||
|
||||
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
|
||||
return ctor ?? null;
|
||||
};
|
||||
|
||||
const floatToInt16 = (sample: number): number => {
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
|
||||
};
|
||||
|
||||
const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
|
||||
if (input.length === 0) {
|
||||
return new Int16Array(0);
|
||||
}
|
||||
if (inputRate === outputRate) {
|
||||
const out = new Int16Array(input.length);
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
out[i] = floatToInt16(input[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const ratio = inputRate / outputRate;
|
||||
const outputLength = Math.max(1, Math.round(input.length / ratio));
|
||||
const out = new Int16Array(outputLength);
|
||||
for (let i = 0; i < outputLength; i++) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const i1 = Math.min(input.length - 1, i0 + 1);
|
||||
const frac = sourceIndex - i0;
|
||||
const sample = input[i0] * (1 - frac) + input[i1] * frac;
|
||||
out[i] = floatToInt16(sample);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): SpeechmaticsAudio {
|
||||
const [isActive, setIsActive] = useState(false);
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
const [isDetecting, setIsDetecting] = useState(false);
|
||||
const [volume, setVolume] = useState(0);
|
||||
const [isMuted, setIsMuted] = useState(false);
|
||||
const [segmentDuration, setSegmentDuration] = useState(0);
|
||||
|
||||
const enableContinuousStreaming = config.enableContinuousStreaming === true;
|
||||
|
||||
const callbacksRef = useRef({
|
||||
onAudioSegment: config.onAudioSegment,
|
||||
onSpeechStart: config.onSpeechStart,
|
||||
onSpeechEnd: config.onSpeechEnd,
|
||||
onError: config.onError,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
callbacksRef.current = {
|
||||
onAudioSegment: config.onAudioSegment,
|
||||
onSpeechStart: config.onSpeechStart,
|
||||
onSpeechEnd: config.onSpeechEnd,
|
||||
onError: config.onError,
|
||||
};
|
||||
}, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd, config.onError]);
|
||||
|
||||
const segmenterRef = useRef<SpeechSegmenter | null>(null);
|
||||
if (segmenterRef.current === null) {
|
||||
segmenterRef.current = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming,
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
|
||||
silenceDurationMs: config.silenceDuration,
|
||||
speechConfirmationMs: config.speechConfirmationDuration,
|
||||
detectionGracePeriodMs: config.detectionGracePeriod,
|
||||
},
|
||||
{
|
||||
onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
|
||||
onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
|
||||
onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
|
||||
onDetectingChange: (next) => setIsDetecting(next),
|
||||
onSpeakingChange: (next) => setIsSpeaking(next),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
segmenterRef.current?.updateConfig({
|
||||
enableContinuousStreaming,
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
|
||||
silenceDurationMs: config.silenceDuration,
|
||||
speechConfirmationMs: config.speechConfirmationDuration,
|
||||
detectionGracePeriodMs: config.detectionGracePeriod,
|
||||
});
|
||||
}, [
|
||||
enableContinuousStreaming,
|
||||
config.volumeThreshold,
|
||||
config.confirmedDropGracePeriod,
|
||||
config.silenceDuration,
|
||||
config.speechConfirmationDuration,
|
||||
config.detectionGracePeriod,
|
||||
]);
|
||||
|
||||
const refs = useRef<{
|
||||
started: boolean;
|
||||
stream: MediaStream | null;
|
||||
context: AudioContext | null;
|
||||
source: MediaStreamAudioSourceNode | null;
|
||||
processor: ScriptProcessorNode | null;
|
||||
gain: GainNode | null;
|
||||
}>({
|
||||
started: false,
|
||||
stream: null,
|
||||
context: null,
|
||||
source: null,
|
||||
processor: null,
|
||||
gain: null,
|
||||
});
|
||||
|
||||
const isMutedRef = useRef(isMuted);
|
||||
useEffect(() => {
|
||||
isMutedRef.current = isMuted;
|
||||
}, [isMuted]);
|
||||
|
||||
const vadConfigRef = useRef({
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
});
|
||||
useEffect(() => {
|
||||
vadConfigRef.current = {
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
};
|
||||
}, [config.volumeThreshold]);
|
||||
|
||||
const vadLogRef = useRef({
|
||||
lastLogMs: 0,
|
||||
lastSpeaking: false,
|
||||
lastDetecting: false,
|
||||
});
|
||||
|
||||
// Update segment duration timer
|
||||
useEffect(() => {
|
||||
if (!isDetecting && !isSpeaking) {
|
||||
setSegmentDuration(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
|
||||
const interval = setInterval(() => {
|
||||
setSegmentDuration(Date.now() - startTime);
|
||||
}, 100);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [isDetecting, isSpeaking]);
|
||||
|
||||
const stopInternal = useCallback(async () => {
|
||||
refs.current.started = false;
|
||||
|
||||
try {
|
||||
refs.current.processor?.disconnect();
|
||||
refs.current.source?.disconnect();
|
||||
refs.current.gain?.disconnect();
|
||||
} catch {
|
||||
// best-effort teardown
|
||||
}
|
||||
|
||||
if (refs.current.stream) {
|
||||
for (const track of refs.current.stream.getTracks()) {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const context = refs.current.context;
|
||||
if (context && context.state !== "closed") {
|
||||
try {
|
||||
await context.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
refs.current.stream = null;
|
||||
refs.current.context = null;
|
||||
refs.current.source = null;
|
||||
refs.current.processor = null;
|
||||
refs.current.gain = null;
|
||||
|
||||
segmenterRef.current?.stop(Date.now());
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
setVolume(0);
|
||||
setIsMuted(false);
|
||||
}, []);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (refs.current.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const missingNavigator =
|
||||
typeof navigator === "undefined" ||
|
||||
!navigator.mediaDevices ||
|
||||
typeof navigator.mediaDevices.getUserMedia !== "function";
|
||||
|
||||
const secureContext =
|
||||
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
|
||||
? window.isSecureContext
|
||||
: true;
|
||||
const currentOrigin = typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
|
||||
const isTauri = getTauri() !== null;
|
||||
|
||||
try {
|
||||
if (missingNavigator) {
|
||||
throw new Error("Microphone capture is not supported in this environment");
|
||||
}
|
||||
console.log("[Voice][Web] Microphone preflight", {
|
||||
secureContext,
|
||||
currentOrigin,
|
||||
isTauri,
|
||||
hasMediaDevices:
|
||||
typeof navigator !== "undefined" &&
|
||||
!!navigator.mediaDevices &&
|
||||
typeof navigator.mediaDevices.getUserMedia === "function",
|
||||
});
|
||||
if (!secureContext && !isTauri) {
|
||||
throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
|
||||
}
|
||||
if (!secureContext && isTauri) {
|
||||
console.warn(
|
||||
"[Voice][Web] Insecure context reported under Tauri; attempting getUserMedia anyway",
|
||||
{ currentOrigin }
|
||||
);
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
if (context.state === "suspended") {
|
||||
await context.resume();
|
||||
}
|
||||
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const processor = context.createScriptProcessor(4096, 1, 1);
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 0;
|
||||
|
||||
refs.current = {
|
||||
started: true,
|
||||
stream,
|
||||
context,
|
||||
source,
|
||||
processor,
|
||||
gain,
|
||||
};
|
||||
|
||||
processor.onaudioprocess = (event) => {
|
||||
if (!refs.current.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
const sample = input[i];
|
||||
sumSquares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||
const normalized = Math.min(1, Math.max(0, rms * 2));
|
||||
setVolume(normalized);
|
||||
|
||||
if (isMutedRef.current) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nowMs = Date.now();
|
||||
segmenterRef.current?.pushVolumeLevel(normalized, nowMs);
|
||||
const segmenter = segmenterRef.current;
|
||||
if (segmenter) {
|
||||
const speakingNow = segmenter.getIsSpeaking();
|
||||
const detectingNow = segmenter.getIsDetecting();
|
||||
const shouldLog =
|
||||
nowMs - vadLogRef.current.lastLogMs >= 150 ||
|
||||
speakingNow !== vadLogRef.current.lastSpeaking ||
|
||||
detectingNow !== vadLogRef.current.lastDetecting;
|
||||
if (shouldLog) {
|
||||
const threshold = vadConfigRef.current.volumeThreshold;
|
||||
const releaseThreshold = segmenter.getVolumeReleaseThreshold();
|
||||
console.log("[Voice][Web][VAD] level", {
|
||||
volume: Number(normalized.toFixed(3)),
|
||||
threshold: Number(threshold.toFixed(3)),
|
||||
releaseThreshold: Number(releaseThreshold.toFixed(3)),
|
||||
confirmedDropGraceMs: config.confirmedDropGracePeriod ?? 250,
|
||||
isSpeaking: speakingNow,
|
||||
isDetecting: detectingNow,
|
||||
});
|
||||
vadLogRef.current.lastLogMs = nowMs;
|
||||
vadLogRef.current.lastSpeaking = speakingNow;
|
||||
vadLogRef.current.lastDetecting = detectingNow;
|
||||
}
|
||||
}
|
||||
|
||||
const pcm16 = resampleToPcm16(input, context.sampleRate, 16000);
|
||||
const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength);
|
||||
segmenterRef.current?.pushPcmChunk(pcmBytes);
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(gain);
|
||||
gain.connect(context.destination);
|
||||
|
||||
setIsActive(true);
|
||||
return;
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
callbacksRef.current.onError?.(err);
|
||||
await stopInternal();
|
||||
throw err;
|
||||
}
|
||||
}, [stopInternal]);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
await stopInternal();
|
||||
}, [stopInternal]);
|
||||
|
||||
const toggleMute = useCallback(() => {
|
||||
setIsMuted((prev) => {
|
||||
const nextMuted = !prev;
|
||||
if (nextMuted) {
|
||||
segmenterRef.current?.reset();
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
}
|
||||
return nextMuted;
|
||||
});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (refs.current.started) {
|
||||
void stopInternal();
|
||||
}
|
||||
};
|
||||
}, [stopInternal]);
|
||||
|
||||
return useMemo(
|
||||
() => ({
|
||||
start,
|
||||
stop,
|
||||
toggleMute,
|
||||
isActive,
|
||||
isSpeaking,
|
||||
isDetecting,
|
||||
isMuted,
|
||||
volume,
|
||||
segmentDuration,
|
||||
}),
|
||||
[start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration]
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user