refactor(app): replace voice hooks with voice runtime and audio engine architecture
This commit is contained in:
Mohamed Boudra
@@ -1,7 +1,7 @@
|
||||
import { useRef, ReactNode, useCallback, useEffect, useMemo } from "react";
|
||||
import { Buffer } from "buffer";
|
||||
import { AppState, Platform } from "react-native";
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import { useAudioPlayer } from "@/hooks/use-audio-player";
|
||||
import { useClientActivity } from "@/hooks/use-client-activity";
|
||||
import { usePushTokenRegistration } from "@/hooks/use-push-token-registration";
|
||||
import { clearArchiveAgentPending } from "@/hooks/use-archive-agent";
|
||||
@@ -29,8 +29,13 @@ import type { DaemonClient } from "@server/client/daemon-client";
|
||||
import { File } from "expo-file-system";
|
||||
import {
|
||||
getHostRuntimeStore,
|
||||
useHostRuntimeSession,
|
||||
useHostRuntimeIsConnected,
|
||||
} from "@/runtime/host-runtime";
|
||||
import {
|
||||
useVoiceAudioEngineOptional,
|
||||
useVoiceRuntimeOptional,
|
||||
} from "@/contexts/voice-context";
|
||||
import type { AudioPlaybackSource } from "@/voice/audio-engine-types";
|
||||
import {
|
||||
useSessionStore,
|
||||
type Agent,
|
||||
@@ -73,6 +78,66 @@ export type {
|
||||
const HISTORY_STALE_AFTER_MS = 60_000;
|
||||
const AUTHORITATIVE_REVALIDATION_DEBOUNCE_MS = 300;
|
||||
|
||||
type AudioOutputPayload = Extract<
|
||||
SessionOutboundMessage,
|
||||
{ type: "audio_output" }
|
||||
>["payload"];
|
||||
|
||||
interface BufferedAudioChunk {
|
||||
chunkIndex: number;
|
||||
audio: string;
|
||||
format: string;
|
||||
id: string;
|
||||
}
|
||||
|
||||
function decodeBase64Chunk(base64: string): Uint8Array {
|
||||
return Buffer.from(base64, "base64");
|
||||
}
|
||||
|
||||
function buildAudioPlaybackSource(
|
||||
chunks: BufferedAudioChunk[]
|
||||
): AudioPlaybackSource {
|
||||
const decodedChunks = chunks.map((chunk) => decodeBase64Chunk(chunk.audio));
|
||||
const totalSize = decodedChunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const output = new Uint8Array(totalSize);
|
||||
let offset = 0;
|
||||
for (const chunk of decodedChunks) {
|
||||
output.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
const format = chunks[0]?.format ?? "pcm";
|
||||
const mimeType =
|
||||
format === "pcm"
|
||||
? "audio/pcm;rate=24000;bits=16"
|
||||
: format === "mp3"
|
||||
? "audio/mpeg"
|
||||
: `audio/${format}`;
|
||||
|
||||
const bytes = output.slice();
|
||||
return {
|
||||
size: bytes.byteLength,
|
||||
type: mimeType,
|
||||
async arrayBuffer() {
|
||||
return bytes.buffer.slice(
|
||||
bytes.byteOffset,
|
||||
bytes.byteOffset + bytes.byteLength
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function logVoiceSession(
|
||||
event: string,
|
||||
details?: Record<string, unknown>
|
||||
): void {
|
||||
if (details) {
|
||||
console.log(`[VoiceSession] ${event}`, details);
|
||||
return;
|
||||
}
|
||||
console.log(`[VoiceSession] ${event}`);
|
||||
}
|
||||
|
||||
const findLatestAssistantMessageText = (items: StreamItem[]): string | null => {
|
||||
for (let i = items.length - 1; i >= 0; i -= 1) {
|
||||
const item = items[i];
|
||||
@@ -206,8 +271,15 @@ function SessionProviderInternal({
|
||||
serverId,
|
||||
client,
|
||||
}: SessionProviderClientProps) {
|
||||
const voiceRuntime = useVoiceRuntimeOptional();
|
||||
const voiceAudioEngine = useVoiceAudioEngineOptional();
|
||||
console.log("[SessionProvider] render", {
|
||||
serverId,
|
||||
hasVoiceRuntime: Boolean(voiceRuntime),
|
||||
hasVoiceAudioEngine: Boolean(voiceAudioEngine),
|
||||
});
|
||||
const queryClient = useQueryClient();
|
||||
const { isConnected } = useHostRuntimeSession(serverId);
|
||||
const isConnected = useHostRuntimeIsConnected(serverId);
|
||||
|
||||
// Zustand store actions
|
||||
const initializeSession = useSessionStore((state) => state.initializeSession);
|
||||
@@ -276,16 +348,6 @@ function SessionProviderInternal({
|
||||
(state) => state.sessions[serverId]?.agents
|
||||
);
|
||||
|
||||
// State for voice detection flags (will be set by RealtimeContext)
|
||||
const isDetectingRef = useRef(false);
|
||||
const isSpeakingRef = useRef(false);
|
||||
|
||||
const audioPlayer = useAudioPlayer({
|
||||
isDetecting: () => isDetectingRef.current,
|
||||
isSpeaking: () => isSpeakingRef.current,
|
||||
});
|
||||
|
||||
const activeAudioGroupsRef = useRef<Set<string>>(new Set());
|
||||
const previousAgentStatusRef = useRef<Map<string, AgentLifecycleStatus>>(
|
||||
new Map()
|
||||
);
|
||||
@@ -308,6 +370,10 @@ function SessionProviderInternal({
|
||||
const revalidationInFlightRef = useRef<Promise<void> | null>(null);
|
||||
const revalidationQueuedRef = useRef(false);
|
||||
const wasConnectedRef = useRef(isConnected);
|
||||
const audioOutputBuffersRef = useRef<Map<string, BufferedAudioChunk[]>>(
|
||||
new Map()
|
||||
);
|
||||
const activeAudioGroupsRef = useRef<Set<string>>(new Set());
|
||||
|
||||
useEffect(() => {
|
||||
const subscription = AppState.addEventListener("change", (nextState) => {
|
||||
@@ -617,24 +683,83 @@ function SessionProviderInternal({
|
||||
[serverId]
|
||||
);
|
||||
|
||||
// Buffer for streaming audio chunks
|
||||
interface AudioChunk {
|
||||
chunkIndex: number;
|
||||
audio: string; // base64
|
||||
format: string;
|
||||
id: string;
|
||||
}
|
||||
const audioChunkBuffersRef = useRef<Map<string, AudioChunk[]>>(new Map());
|
||||
|
||||
// Initialize session in store
|
||||
useEffect(() => {
|
||||
initializeSession(serverId, client, audioPlayer);
|
||||
}, [serverId, client, audioPlayer, initializeSession]);
|
||||
console.log("[SessionProvider] mount", { serverId });
|
||||
return () => {
|
||||
console.log("[SessionProvider] unmount", { serverId });
|
||||
};
|
||||
}, [serverId]);
|
||||
|
||||
useEffect(() => {
|
||||
initializeSession(serverId, client);
|
||||
}, [serverId, client, initializeSession]);
|
||||
|
||||
useEffect(() => {
|
||||
updateSessionClient(serverId, client);
|
||||
}, [serverId, client, updateSessionClient]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!voiceRuntime) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("[SessionProvider] register_voice_session", { serverId });
|
||||
return voiceRuntime.registerSession({
|
||||
serverId,
|
||||
setVoiceMode: async (enabled, agentId) => {
|
||||
console.log("[SessionProvider] setVoiceMode", {
|
||||
serverId,
|
||||
enabled,
|
||||
agentId: agentId ?? null,
|
||||
});
|
||||
if (!client) {
|
||||
throw new Error("Daemon unavailable");
|
||||
}
|
||||
await client.setVoiceMode(enabled, agentId);
|
||||
},
|
||||
sendVoiceAudioChunk: async (audioData, mimeType, isLast) => {
|
||||
console.log("[SessionProvider] sendVoiceAudioChunk", {
|
||||
serverId,
|
||||
audioDataLength: audioData.length,
|
||||
mimeType,
|
||||
isLast,
|
||||
});
|
||||
if (!client) {
|
||||
throw new Error("Daemon unavailable");
|
||||
}
|
||||
await client.sendVoiceAudioChunk(audioData, mimeType, isLast);
|
||||
},
|
||||
abortRequest: async () => {
|
||||
console.log("[SessionProvider] abortRequest", { serverId });
|
||||
if (!client) {
|
||||
throw new Error("Daemon unavailable");
|
||||
}
|
||||
await client.abortRequest();
|
||||
},
|
||||
setAssistantAudioPlaying: (isPlaying) => {
|
||||
console.log("[SessionProvider] setAssistantAudioPlaying", {
|
||||
serverId,
|
||||
isPlaying,
|
||||
});
|
||||
setIsPlayingAudio(serverId, isPlaying);
|
||||
},
|
||||
});
|
||||
}, [
|
||||
client,
|
||||
serverId,
|
||||
setIsPlayingAudio,
|
||||
voiceRuntime,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
console.log("[SessionProvider] updateSessionConnection", {
|
||||
serverId,
|
||||
isConnected,
|
||||
});
|
||||
voiceRuntime?.updateSessionConnection(serverId, isConnected);
|
||||
}, [isConnected, serverId, voiceRuntime]);
|
||||
|
||||
// If the client drops mid-initialization, clear pending flags
|
||||
useEffect(() => {
|
||||
if (!isConnected) {
|
||||
@@ -962,6 +1087,14 @@ function SessionProviderInternal({
|
||||
const { agentId, event, timestamp, seq, epoch } = message.payload;
|
||||
const parsedTimestamp = new Date(timestamp);
|
||||
const streamEvent = event as AgentStreamEventPayload;
|
||||
if (
|
||||
event.type === "turn_started" ||
|
||||
event.type === "turn_completed" ||
|
||||
event.type === "turn_failed" ||
|
||||
event.type === "turn_canceled"
|
||||
) {
|
||||
voiceRuntime?.onTurnEvent(serverId, agentId, event.type);
|
||||
}
|
||||
|
||||
// Attention notification stays in React (not extractable to pure reducer)
|
||||
if (event.type === "attention_required") {
|
||||
@@ -1161,92 +1294,102 @@ function SessionProviderInternal({
|
||||
|
||||
const unsubAudioOutput = client.on("audio_output", async (message) => {
|
||||
if (message.type !== "audio_output") return;
|
||||
const data = message.payload;
|
||||
const playbackGroupId = data.groupId ?? data.id;
|
||||
const isFinalChunk = data.isLastChunk ?? true;
|
||||
const chunkIndex = data.chunkIndex ?? 0;
|
||||
if (!voiceAudioEngine) {
|
||||
return;
|
||||
}
|
||||
|
||||
const payload: AudioOutputPayload = message.payload;
|
||||
console.log("[SessionProvider] audio_output", {
|
||||
serverId,
|
||||
id: payload.id,
|
||||
groupId: payload.groupId ?? null,
|
||||
chunkIndex: payload.chunkIndex ?? 0,
|
||||
isLastChunk: payload.isLastChunk ?? true,
|
||||
format: payload.format,
|
||||
isVoiceMode: payload.isVoiceMode ?? false,
|
||||
audioLength: payload.audio.length,
|
||||
});
|
||||
const playbackGroupId = payload.groupId ?? payload.id;
|
||||
const chunkIndex = payload.chunkIndex ?? 0;
|
||||
const isFinalChunk = payload.isLastChunk ?? true;
|
||||
|
||||
if (!audioOutputBuffersRef.current.has(playbackGroupId)) {
|
||||
audioOutputBuffersRef.current.set(playbackGroupId, []);
|
||||
}
|
||||
|
||||
const bufferedChunks = audioOutputBuffersRef.current.get(playbackGroupId)!;
|
||||
bufferedChunks.push({
|
||||
chunkIndex,
|
||||
audio: payload.audio,
|
||||
format: payload.format,
|
||||
id: payload.id,
|
||||
});
|
||||
|
||||
activeAudioGroupsRef.current.add(playbackGroupId);
|
||||
setIsPlayingAudio(serverId, true);
|
||||
|
||||
if (!audioChunkBuffersRef.current.has(playbackGroupId)) {
|
||||
audioChunkBuffersRef.current.set(playbackGroupId, []);
|
||||
}
|
||||
|
||||
const buffer = audioChunkBuffersRef.current.get(playbackGroupId)!;
|
||||
buffer.push({
|
||||
chunkIndex,
|
||||
audio: data.audio,
|
||||
format: data.format,
|
||||
id: data.id,
|
||||
});
|
||||
|
||||
if (!isFinalChunk) {
|
||||
return;
|
||||
}
|
||||
buffer.sort((a, b) => a.chunkIndex - b.chunkIndex);
|
||||
|
||||
let playbackFailed = false;
|
||||
const chunkIds = buffer.map((chunk) => chunk.id);
|
||||
|
||||
const confirmAudioPlayed = (ids: string[]) => {
|
||||
if (!client) {
|
||||
console.warn("[Session] audio_played skipped: daemon unavailable");
|
||||
return;
|
||||
}
|
||||
ids.forEach((chunkId) => {
|
||||
void client.audioPlayed(chunkId).catch((error) => {
|
||||
console.warn("[Session] Failed to confirm audio playback:", error);
|
||||
});
|
||||
bufferedChunks.sort((left, right) => left.chunkIndex - right.chunkIndex);
|
||||
const chunkIds = bufferedChunks.map((chunk) => chunk.id);
|
||||
const shouldPlay =
|
||||
!payload.isVoiceMode ||
|
||||
(voiceRuntime?.shouldPlayVoiceAudio(serverId) ?? false);
|
||||
const audioBlob = buildAudioPlaybackSource(bufferedChunks);
|
||||
if (payload.isVoiceMode) {
|
||||
logVoiceSession("audio_output_ready", {
|
||||
serverId,
|
||||
playbackGroupId,
|
||||
chunkCount: bufferedChunks.length,
|
||||
format: payload.format,
|
||||
blobBytes: audioBlob.size,
|
||||
shouldPlay,
|
||||
});
|
||||
}
|
||||
const confirmAudioPlayed = async () => {
|
||||
await Promise.all(
|
||||
chunkIds.map((chunkId) =>
|
||||
client.audioPlayed(chunkId).catch((error) => {
|
||||
console.warn("[Session] Failed to confirm audio playback:", error);
|
||||
})
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
let startedVoicePlayback = false;
|
||||
try {
|
||||
const mimeType =
|
||||
data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`;
|
||||
|
||||
const decodedChunks: Uint8Array[] = [];
|
||||
let totalSize = 0;
|
||||
|
||||
for (const chunk of buffer) {
|
||||
const binaryString = atob(chunk.audio);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
if (shouldPlay) {
|
||||
if (payload.isVoiceMode) {
|
||||
startedVoicePlayback = true;
|
||||
voiceRuntime?.onAssistantAudioStarted(serverId);
|
||||
}
|
||||
decodedChunks.push(bytes);
|
||||
totalSize += bytes.length;
|
||||
await voiceAudioEngine.play(audioBlob);
|
||||
if (payload.isVoiceMode) {
|
||||
logVoiceSession("audio_output_played", {
|
||||
serverId,
|
||||
playbackGroupId,
|
||||
chunkCount: bufferedChunks.length,
|
||||
});
|
||||
}
|
||||
} else if (payload.isVoiceMode) {
|
||||
logVoiceSession("audio_output_suppressed", {
|
||||
serverId,
|
||||
playbackGroupId,
|
||||
});
|
||||
}
|
||||
|
||||
const concatenatedBytes = new Uint8Array(totalSize);
|
||||
let offset = 0;
|
||||
for (const chunk of decodedChunks) {
|
||||
concatenatedBytes.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
const audioBlob = {
|
||||
type: mimeType,
|
||||
size: totalSize,
|
||||
arrayBuffer: async () => {
|
||||
return concatenatedBytes.buffer;
|
||||
},
|
||||
} as Blob;
|
||||
|
||||
await audioPlayer.play(audioBlob);
|
||||
|
||||
confirmAudioPlayed(chunkIds);
|
||||
} catch (error: any) {
|
||||
playbackFailed = true;
|
||||
await confirmAudioPlayed();
|
||||
} catch (error) {
|
||||
console.error("[Session] Audio playback error:", error);
|
||||
|
||||
confirmAudioPlayed(chunkIds);
|
||||
await confirmAudioPlayed();
|
||||
} finally {
|
||||
audioChunkBuffersRef.current.delete(playbackGroupId);
|
||||
audioOutputBuffersRef.current.delete(playbackGroupId);
|
||||
activeAudioGroupsRef.current.delete(playbackGroupId);
|
||||
setIsPlayingAudio(serverId, activeAudioGroupsRef.current.size > 0);
|
||||
|
||||
if (activeAudioGroupsRef.current.size === 0) {
|
||||
setIsPlayingAudio(serverId, false);
|
||||
if (startedVoicePlayback) {
|
||||
voiceRuntime?.onAssistantAudioFinished(serverId);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1254,6 +1397,12 @@ function SessionProviderInternal({
|
||||
const unsubActivity = client.on("activity_log", (message) => {
|
||||
if (message.type !== "activity_log") return;
|
||||
const data = message.payload;
|
||||
console.log("[SessionProvider] activity_log", {
|
||||
serverId,
|
||||
type: data.type,
|
||||
contentPreview:
|
||||
typeof data.content === "string" ? data.content.slice(0, 80) : null,
|
||||
});
|
||||
|
||||
if (data.type === "system" && data.content.includes("Transcribing")) {
|
||||
return;
|
||||
@@ -1374,13 +1523,18 @@ function SessionProviderInternal({
|
||||
if (message.type !== "transcription_result") return;
|
||||
|
||||
const transcriptText = message.payload.text.trim();
|
||||
console.log("[SessionProvider] transcription_result", {
|
||||
serverId,
|
||||
textLength: transcriptText.length,
|
||||
textPreview: transcriptText.slice(0, 80),
|
||||
});
|
||||
|
||||
if (!transcriptText) {
|
||||
} else {
|
||||
audioPlayer.stop();
|
||||
setIsPlayingAudio(serverId, false);
|
||||
setCurrentAssistantMessage(serverId, "");
|
||||
voiceRuntime?.onTranscriptionResult(serverId, transcriptText);
|
||||
return;
|
||||
}
|
||||
|
||||
setCurrentAssistantMessage(serverId, "");
|
||||
});
|
||||
|
||||
const unsubAgentDeleted = client.on("agent_deleted", (message) => {
|
||||
@@ -1498,7 +1652,6 @@ function SessionProviderInternal({
|
||||
};
|
||||
}, [
|
||||
client,
|
||||
audioPlayer,
|
||||
queryClient,
|
||||
serverId,
|
||||
setIsPlayingAudio,
|
||||
@@ -1521,6 +1674,8 @@ function SessionProviderInternal({
|
||||
requestCanonicalCatchUp,
|
||||
applyAgentUpdatePayload,
|
||||
applyTimelineResponse,
|
||||
voiceRuntime,
|
||||
voiceAudioEngine,
|
||||
]);
|
||||
|
||||
const sendAgentMessage = useCallback(
|
||||
@@ -1742,20 +1897,12 @@ function SessionProviderInternal({
|
||||
[client]
|
||||
);
|
||||
|
||||
const setVoiceDetectionFlags = useCallback(
|
||||
(isDetecting: boolean, isSpeaking: boolean) => {
|
||||
isDetectingRef.current = isDetecting;
|
||||
isSpeakingRef.current = isSpeaking;
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
clearSession(serverId);
|
||||
};
|
||||
}, [serverId, clearSession]);
|
||||
}, [clearSession, serverId]);
|
||||
|
||||
return children;
|
||||
}
|
||||
|
||||
@@ -1,41 +1,119 @@
|
||||
import { createContext, useContext, useState, ReactNode, useCallback, useEffect, useRef } from "react";
|
||||
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
|
||||
import type { SessionState } from "@/stores/session-store";
|
||||
import { useSessionStore } from "@/stores/session-store";
|
||||
import { useHostRuntimeSession } from "@/runtime/host-runtime";
|
||||
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
|
||||
import {
|
||||
createContext,
|
||||
useContext,
|
||||
useEffect,
|
||||
useRef,
|
||||
useSyncExternalStore,
|
||||
type ReactNode,
|
||||
} from "react";
|
||||
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
|
||||
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
|
||||
import { useSessionStore } from "@/stores/session-store";
|
||||
import { createAudioEngine } from "@/voice/audio-engine";
|
||||
import type { AudioEngine } from "@/voice/audio-engine-types";
|
||||
import {
|
||||
createVoiceRuntime,
|
||||
type VoiceRuntime,
|
||||
type VoiceRuntimeSnapshot,
|
||||
type VoiceRuntimeTelemetrySnapshot,
|
||||
} from "@/voice/voice-runtime";
|
||||
|
||||
const KEEP_AWAKE_TAG = "paseo:voice";
|
||||
interface VoiceContextValue {
|
||||
isVoiceMode: boolean;
|
||||
isVoiceSwitching: boolean;
|
||||
volume: number;
|
||||
isMuted: boolean;
|
||||
isDetecting: boolean;
|
||||
isSpeaking: boolean;
|
||||
segmentDuration: number;
|
||||
interface VoiceContextValue extends VoiceRuntimeSnapshot {
|
||||
startVoice: (serverId: string, agentId: string) => Promise<void>;
|
||||
stopVoice: () => Promise<void>;
|
||||
isVoiceModeForAgent: (serverId: string, agentId: string) => boolean;
|
||||
toggleMute: () => void;
|
||||
activeServerId: string | null;
|
||||
activeAgentId: string | null;
|
||||
}
|
||||
|
||||
const VoiceContext = createContext<VoiceContextValue | null>(null);
|
||||
const EMPTY_SNAPSHOT: VoiceRuntimeSnapshot = {
|
||||
phase: "disabled",
|
||||
isVoiceMode: false,
|
||||
isVoiceSwitching: false,
|
||||
isMuted: false,
|
||||
activeServerId: null,
|
||||
activeAgentId: null,
|
||||
};
|
||||
|
||||
const EMPTY_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
|
||||
volume: 0,
|
||||
isDetecting: false,
|
||||
isSpeaking: false,
|
||||
segmentDuration: 0,
|
||||
};
|
||||
|
||||
const VoiceRuntimeContext = createContext<VoiceRuntime | null>(null);
|
||||
const VoiceAudioEngineContext = createContext<AudioEngine | null>(null);
|
||||
|
||||
const noopSubscribe = () => () => {};
|
||||
const getEmptySnapshot = () => EMPTY_SNAPSHOT;
|
||||
const getEmptyTelemetry = () => EMPTY_TELEMETRY;
|
||||
let nextVoiceProviderInstanceId = 1;
|
||||
|
||||
function getProviderTraceStack(): string | undefined {
|
||||
const stack = new Error().stack;
|
||||
if (!stack) {
|
||||
return undefined;
|
||||
}
|
||||
return stack
|
||||
.split("\n")
|
||||
.slice(2, 7)
|
||||
.map((line) => line.trim())
|
||||
.join(" | ");
|
||||
}
|
||||
|
||||
export function useVoice() {
|
||||
const context = useContext(VoiceContext);
|
||||
if (!context) {
|
||||
const value = useVoiceOptional();
|
||||
if (!value) {
|
||||
throw new Error("useVoice must be used within VoiceProvider");
|
||||
}
|
||||
return context;
|
||||
return value;
|
||||
}
|
||||
|
||||
export function useVoiceOptional(): VoiceContextValue | null {
|
||||
return useContext(VoiceContext);
|
||||
const runtime = useContext(VoiceRuntimeContext);
|
||||
const snapshot = useSyncExternalStore(
|
||||
runtime ? runtime.subscribe : noopSubscribe,
|
||||
runtime ? runtime.getSnapshot : getEmptySnapshot,
|
||||
runtime ? runtime.getSnapshot : getEmptySnapshot
|
||||
);
|
||||
|
||||
if (!runtime) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...snapshot,
|
||||
startVoice: runtime.startVoice,
|
||||
stopVoice: runtime.stopVoice,
|
||||
isVoiceModeForAgent: runtime.isVoiceModeForAgent,
|
||||
toggleMute: runtime.toggleMute,
|
||||
};
|
||||
}
|
||||
|
||||
export function useVoiceTelemetry() {
|
||||
const telemetry = useVoiceTelemetryOptional();
|
||||
if (!telemetry) {
|
||||
throw new Error("useVoiceTelemetry must be used within VoiceProvider");
|
||||
}
|
||||
return telemetry;
|
||||
}
|
||||
|
||||
export function useVoiceTelemetryOptional(): VoiceRuntimeTelemetrySnapshot | null {
|
||||
const runtime = useContext(VoiceRuntimeContext);
|
||||
const snapshot = useSyncExternalStore(
|
||||
runtime ? runtime.subscribeTelemetry : noopSubscribe,
|
||||
runtime ? runtime.getTelemetrySnapshot : getEmptyTelemetry,
|
||||
runtime ? runtime.getTelemetrySnapshot : getEmptyTelemetry
|
||||
);
|
||||
|
||||
return runtime ? snapshot : null;
|
||||
}
|
||||
|
||||
export function useVoiceRuntimeOptional(): VoiceRuntime | null {
|
||||
return useContext(VoiceRuntimeContext);
|
||||
}
|
||||
|
||||
export function useVoiceAudioEngineOptional(): AudioEngine | null {
|
||||
return useContext(VoiceAudioEngineContext);
|
||||
}
|
||||
|
||||
interface VoiceProviderProps {
|
||||
@@ -43,405 +121,87 @@ interface VoiceProviderProps {
|
||||
}
|
||||
|
||||
export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||
const getSession = useSessionStore((state) => state.getSession);
|
||||
const [activeServerId, setActiveServerId] = useState<string | null>(null);
|
||||
const [activeAgentId, setActiveAgentId] = useState<string | null>(null);
|
||||
const { client: runtimeClient, isConnected: activeRuntimeConnected } = useHostRuntimeSession(
|
||||
activeServerId ?? ""
|
||||
);
|
||||
const activeSession = useSessionStore(
|
||||
useCallback(
|
||||
(state: ReturnType<typeof useSessionStore.getState>) => {
|
||||
if (!activeServerId) {
|
||||
return null;
|
||||
}
|
||||
return state.sessions[activeServerId] ?? null;
|
||||
const providerIdRef = useRef<number | null>(null);
|
||||
if (providerIdRef.current === null) {
|
||||
providerIdRef.current = nextVoiceProviderInstanceId++;
|
||||
console.log("[VoiceProvider] instance_created", {
|
||||
providerId: providerIdRef.current,
|
||||
stack: getProviderTraceStack(),
|
||||
});
|
||||
}
|
||||
|
||||
const providerId = providerIdRef.current;
|
||||
const engineRef = useRef<AudioEngine | null>(null);
|
||||
const runtimeRef = useRef<VoiceRuntime | null>(null);
|
||||
console.log("[VoiceProvider] render", {
|
||||
providerId,
|
||||
hasEngine: Boolean(engineRef.current),
|
||||
hasRuntime: Boolean(runtimeRef.current),
|
||||
});
|
||||
|
||||
if (!engineRef.current) {
|
||||
let runtime: VoiceRuntime | null = null;
|
||||
console.log("[VoiceProvider] create_engine_and_runtime");
|
||||
const engine = createAudioEngine({
|
||||
onCaptureData: (pcm) => {
|
||||
console.log("[VoiceProvider] onCaptureData", {
|
||||
providerId,
|
||||
bytes: pcm.byteLength,
|
||||
});
|
||||
runtime?.handleCapturePcm(pcm);
|
||||
},
|
||||
[activeServerId]
|
||||
)
|
||||
);
|
||||
const realtimeSessionRef = useRef<SessionState | null>(null);
|
||||
const [isVoiceMode, setIsVoiceMode] = useState(false);
|
||||
const [isVoiceSwitching, setIsVoiceSwitching] = useState(false);
|
||||
const bargeInPlaybackStopRef = useRef<number | null>(null);
|
||||
const wasVoiceSocketConnectedRef = useRef(false);
|
||||
const lastVoiceModeSyncedClientRef = useRef<SessionState["client"] | null>(null);
|
||||
const voiceTransportReadyRef = useRef(false);
|
||||
const voiceResyncInFlightRef = useRef(false);
|
||||
const silenceGraceStartMsRef = useRef<number | null>(null);
|
||||
const speechInterruptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const speechInterruptStartMsRef = useRef<number | null>(null);
|
||||
const speechStartInterruptSentRef = useRef(false);
|
||||
const isVoiceModeRef = useRef(false);
|
||||
const vadStateRef = useRef<{ isDetecting: boolean; isSpeaking: boolean }>({
|
||||
isDetecting: false,
|
||||
isSpeaking: false,
|
||||
});
|
||||
|
||||
const clearSpeechStartInterruptTimer = useCallback((reason: string) => {
|
||||
const timer = speechInterruptTimerRef.current;
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
speechInterruptTimerRef.current = null;
|
||||
const startedAt = speechInterruptStartMsRef.current;
|
||||
speechInterruptStartMsRef.current = null;
|
||||
if (startedAt !== null) {
|
||||
console.log("[Voice] Cleared speech-start interrupt timer", {
|
||||
reason,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
onVolumeLevel: (level) => {
|
||||
console.log("[VoiceProvider] onVolumeLevel", {
|
||||
providerId,
|
||||
level,
|
||||
});
|
||||
} else {
|
||||
console.log("[Voice] Cleared speech-start interrupt timer", { reason });
|
||||
}
|
||||
return;
|
||||
}
|
||||
speechInterruptStartMsRef.current = null;
|
||||
}, []);
|
||||
runtime?.handleCaptureVolume(level);
|
||||
},
|
||||
onError: (error) => {
|
||||
console.error("[VoiceEngine] Capture error:", error);
|
||||
},
|
||||
}, {
|
||||
traceLabel: `voice-provider:${providerId}`,
|
||||
});
|
||||
|
||||
const interruptActiveVoiceTurn = useCallback((source: string) => {
|
||||
const session = realtimeSessionRef.current;
|
||||
const sessionAudioPlayer = session?.audioPlayer ?? null;
|
||||
const sessionClient = session?.client ?? null;
|
||||
const sessionIsPlayingAudio = session?.isPlayingAudio ?? false;
|
||||
runtime = createVoiceRuntime({
|
||||
engine,
|
||||
getServerInfo: (serverId) =>
|
||||
useSessionStore.getState().getSession(serverId)?.serverInfo ?? null,
|
||||
activateKeepAwake: async (tag) => {
|
||||
await activateKeepAwakeAsync(tag);
|
||||
},
|
||||
deactivateKeepAwake: async (tag) => {
|
||||
await deactivateKeepAwake(tag);
|
||||
},
|
||||
});
|
||||
|
||||
if (sessionIsPlayingAudio && sessionAudioPlayer) {
|
||||
if (bargeInPlaybackStopRef.current === null) {
|
||||
bargeInPlaybackStopRef.current = Date.now();
|
||||
}
|
||||
sessionAudioPlayer.stop();
|
||||
}
|
||||
engineRef.current = engine;
|
||||
runtimeRef.current = runtime;
|
||||
}
|
||||
|
||||
try {
|
||||
if (sessionClient) {
|
||||
void sessionClient.abortRequest().catch((error) => {
|
||||
console.error("[Voice] Failed to send abort_request:", error);
|
||||
});
|
||||
}
|
||||
console.log("[Voice] Sent abort_request before streaming audio", { source });
|
||||
} catch (error) {
|
||||
console.error("[Voice] Failed to send abort_request:", error);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const realtimeAudio = useSpeechmaticsAudio({
|
||||
onSpeechStart: () => {
|
||||
console.log("[Voice] Segment started (speech confirmed)");
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
const silenceMs =
|
||||
silenceGraceStartMsRef.current === null
|
||||
? null
|
||||
: Date.now() - silenceGraceStartMsRef.current;
|
||||
if (silenceMs === null) {
|
||||
console.log("[Voice] Segment finalized");
|
||||
} else {
|
||||
console.log("[Voice] Segment finalized", { silenceMs });
|
||||
}
|
||||
silenceGraceStartMsRef.current = null;
|
||||
clearSpeechStartInterruptTimer("speech ended");
|
||||
speechStartInterruptSentRef.current = false;
|
||||
},
|
||||
onAudioSegment: ({ audioData, isLast }) => {
|
||||
if (!voiceTransportReadyRef.current) {
|
||||
console.log("[Voice] Skipping audio segment: voice transport not ready");
|
||||
return;
|
||||
}
|
||||
console.log(
|
||||
"[Voice] Sending audio segment, length:",
|
||||
audioData.length,
|
||||
"isLast:",
|
||||
isLast
|
||||
);
|
||||
|
||||
// Send audio segment to server (realtime always goes to orchestrator)
|
||||
const session = realtimeSessionRef.current;
|
||||
try {
|
||||
if (session?.client) {
|
||||
void session.client
|
||||
.sendVoiceAudioChunk(
|
||||
audioData,
|
||||
"audio/pcm;rate=16000;bits=16",
|
||||
isLast
|
||||
)
|
||||
.catch((error) => {
|
||||
console.error("[Voice] Failed to send audio segment:", error);
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[Voice] Failed to send audio segment:", error);
|
||||
}
|
||||
},
|
||||
onError: (error) => {
|
||||
console.error("[Voice] Audio error:", error);
|
||||
const session = realtimeSessionRef.current;
|
||||
if (session?.client) {
|
||||
// Send error through websocket instead of directly manipulating messages
|
||||
console.error("[Voice] Cannot handle error - setMessages not available from SessionState");
|
||||
}
|
||||
},
|
||||
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
|
||||
confirmedDropGracePeriod: REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
|
||||
silenceDuration: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
|
||||
speechConfirmationDuration: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
|
||||
detectionGracePeriod: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
realtimeSessionRef.current = activeSession;
|
||||
}, [activeSession]);
|
||||
|
||||
useEffect(() => {
|
||||
isVoiceModeRef.current = isVoiceMode;
|
||||
}, [isVoiceMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isVoiceMode) {
|
||||
clearSpeechStartInterruptTimer("voice mode disabled");
|
||||
speechStartInterruptSentRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
if (realtimeAudio.isSpeaking) {
|
||||
if (
|
||||
speechStartInterruptSentRef.current ||
|
||||
speechInterruptTimerRef.current !== null
|
||||
) {
|
||||
return;
|
||||
}
|
||||
speechInterruptStartMsRef.current = Date.now();
|
||||
speechInterruptTimerRef.current = setTimeout(() => {
|
||||
speechInterruptTimerRef.current = null;
|
||||
speechInterruptStartMsRef.current = null;
|
||||
if (
|
||||
!isVoiceModeRef.current ||
|
||||
!vadStateRef.current.isSpeaking ||
|
||||
speechStartInterruptSentRef.current
|
||||
) {
|
||||
return;
|
||||
}
|
||||
speechStartInterruptSentRef.current = true;
|
||||
console.log("[Voice] Speech persisted beyond grace; interrupting turn", {
|
||||
graceMs: REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs,
|
||||
});
|
||||
interruptActiveVoiceTurn("speech_start_grace_elapsed");
|
||||
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
|
||||
return;
|
||||
}
|
||||
|
||||
clearSpeechStartInterruptTimer("speech stopped before grace");
|
||||
}, [
|
||||
clearSpeechStartInterruptTimer,
|
||||
interruptActiveVoiceTurn,
|
||||
isVoiceMode,
|
||||
realtimeAudio.isSpeaking,
|
||||
]);
|
||||
const engine = engineRef.current!;
|
||||
const runtime = runtimeRef.current!;
|
||||
|
||||
useEffect(() => {
|
||||
console.log("[VoiceProvider] mount", {
|
||||
providerId,
|
||||
});
|
||||
return () => {
|
||||
clearSpeechStartInterruptTimer("voice provider unmounted");
|
||||
};
|
||||
}, [clearSpeechStartInterruptTimer]);
|
||||
|
||||
useEffect(() => {
|
||||
const next = {
|
||||
isDetecting: realtimeAudio.isDetecting,
|
||||
isSpeaking: realtimeAudio.isSpeaking,
|
||||
};
|
||||
const prev = vadStateRef.current;
|
||||
|
||||
if (!isVoiceMode) {
|
||||
silenceGraceStartMsRef.current = null;
|
||||
vadStateRef.current = next;
|
||||
return;
|
||||
}
|
||||
|
||||
// Confirmed speech dropped below threshold: grace window starts.
|
||||
if (prev.isSpeaking && !next.isSpeaking && next.isDetecting) {
|
||||
silenceGraceStartMsRef.current = Date.now();
|
||||
console.log("[Voice] Grace started (speech dropped below threshold)");
|
||||
}
|
||||
|
||||
// User resumed speaking before grace timeout elapsed.
|
||||
if (!prev.isSpeaking && next.isSpeaking && silenceGraceStartMsRef.current !== null) {
|
||||
const resumedAfterMs = Date.now() - silenceGraceStartMsRef.current;
|
||||
console.log("[Voice] Speech resumed during grace", { resumedAfterMs });
|
||||
silenceGraceStartMsRef.current = null;
|
||||
}
|
||||
|
||||
// Fully idle (neither detecting nor speaking).
|
||||
if (!next.isDetecting && !next.isSpeaking) {
|
||||
silenceGraceStartMsRef.current = null;
|
||||
speechStartInterruptSentRef.current = false;
|
||||
}
|
||||
|
||||
vadStateRef.current = next;
|
||||
}, [isVoiceMode, realtimeAudio.isDetecting, realtimeAudio.isSpeaking]);
|
||||
|
||||
useEffect(() => {
|
||||
const connected = activeRuntimeConnected;
|
||||
const client = runtimeClient;
|
||||
if (!connected) {
|
||||
voiceTransportReadyRef.current = false;
|
||||
}
|
||||
|
||||
if (!isVoiceMode || !activeServerId || !activeAgentId || !client) {
|
||||
wasVoiceSocketConnectedRef.current = connected;
|
||||
if (!isVoiceMode) {
|
||||
lastVoiceModeSyncedClientRef.current = null;
|
||||
}
|
||||
voiceTransportReadyRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
const connectionRecovered = connected && !wasVoiceSocketConnectedRef.current;
|
||||
const clientChanged = lastVoiceModeSyncedClientRef.current !== client;
|
||||
if (connected && (connectionRecovered || clientChanged)) {
|
||||
if (!voiceResyncInFlightRef.current) {
|
||||
voiceResyncInFlightRef.current = true;
|
||||
voiceTransportReadyRef.current = false;
|
||||
setIsVoiceSwitching(true);
|
||||
void client.setVoiceMode(true, activeAgentId).then(
|
||||
() => {
|
||||
console.log("[Voice] Re-synced voice mode after reconnect");
|
||||
lastVoiceModeSyncedClientRef.current = client;
|
||||
voiceTransportReadyRef.current = true;
|
||||
},
|
||||
(error) => {
|
||||
console.error("[Voice] Failed to re-sync voice mode:", error);
|
||||
}
|
||||
).finally(() => {
|
||||
voiceResyncInFlightRef.current = false;
|
||||
setIsVoiceSwitching(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
wasVoiceSocketConnectedRef.current = connected;
|
||||
}, [activeAgentId, activeRuntimeConnected, activeServerId, isVoiceMode, runtimeClient]);
|
||||
|
||||
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
|
||||
|
||||
useEffect(() => {
|
||||
if (!isPlayingAudio && bargeInPlaybackStopRef.current !== null) {
|
||||
const latencyMs = Date.now() - bargeInPlaybackStopRef.current;
|
||||
console.log("[Telemetry] barge_in.playback_stop_latency", {
|
||||
latencyMs,
|
||||
startedAt: new Date(bargeInPlaybackStopRef.current).toISOString(),
|
||||
completedAt: new Date().toISOString(),
|
||||
console.log("[VoiceProvider] unmount", {
|
||||
providerId,
|
||||
});
|
||||
bargeInPlaybackStopRef.current = null;
|
||||
}
|
||||
}, [isPlayingAudio]);
|
||||
|
||||
const startVoice = useCallback(
|
||||
async (serverId: string, agentId: string) => {
|
||||
const session = getSession(serverId) ?? null;
|
||||
if (!session) {
|
||||
throw new Error(`Host ${serverId} is not connected`);
|
||||
}
|
||||
const unavailableMessage = resolveVoiceUnavailableMessage({
|
||||
serverInfo: session.serverInfo,
|
||||
mode: "voice",
|
||||
void runtime.destroy().catch((error) => {
|
||||
console.error("[VoiceProvider] Failed to destroy voice runtime", error);
|
||||
});
|
||||
if (unavailableMessage) {
|
||||
throw new Error(unavailableMessage);
|
||||
}
|
||||
|
||||
setIsVoiceSwitching(true);
|
||||
voiceTransportReadyRef.current = false;
|
||||
try {
|
||||
const previousSession = realtimeSessionRef.current;
|
||||
if (
|
||||
isVoiceMode &&
|
||||
previousSession?.client &&
|
||||
(activeServerId !== serverId || activeAgentId !== agentId)
|
||||
) {
|
||||
await previousSession.client.setVoiceMode(false);
|
||||
}
|
||||
|
||||
realtimeSessionRef.current = session;
|
||||
setActiveServerId(serverId);
|
||||
setActiveAgentId(agentId);
|
||||
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
|
||||
console.warn("[Voice] Failed to activate keep-awake:", error);
|
||||
});
|
||||
if (session?.client) {
|
||||
await session.client.setVoiceMode(true, agentId);
|
||||
} else {
|
||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||
}
|
||||
await session.audioPlayer?.warmup?.();
|
||||
await realtimeAudio.start();
|
||||
voiceTransportReadyRef.current = true;
|
||||
setIsVoiceMode(true);
|
||||
lastVoiceModeSyncedClientRef.current = session.client;
|
||||
console.log("[Voice] Mode enabled");
|
||||
} catch (error: any) {
|
||||
console.error("[Voice] Failed to start:", error);
|
||||
await realtimeAudio.stop().catch(() => undefined);
|
||||
setActiveServerId((current) => (current === serverId ? null : current));
|
||||
setActiveAgentId((current) => (current === agentId ? null : current));
|
||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsVoiceSwitching(false);
|
||||
}
|
||||
},
|
||||
[activeAgentId, activeServerId, getSession, isVoiceMode, realtimeAudio]
|
||||
);
|
||||
|
||||
const stopVoice = useCallback(async () => {
|
||||
setIsVoiceSwitching(true);
|
||||
voiceTransportReadyRef.current = false;
|
||||
try {
|
||||
const session = realtimeSessionRef.current;
|
||||
session?.audioPlayer?.stop();
|
||||
if (session?.client) {
|
||||
await session.client.setVoiceMode(false);
|
||||
lastVoiceModeSyncedClientRef.current = session.client;
|
||||
} else {
|
||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||
}
|
||||
await realtimeAudio.stop();
|
||||
setIsVoiceMode(false);
|
||||
setActiveServerId(null);
|
||||
setActiveAgentId(null);
|
||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
console.log("[Voice] Mode disabled");
|
||||
} catch (error: any) {
|
||||
console.error("[Voice] Failed to stop:", error);
|
||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsVoiceSwitching(false);
|
||||
}
|
||||
}, [realtimeAudio]);
|
||||
|
||||
const isVoiceModeForAgent = useCallback(
|
||||
(serverId: string, agentId: string) =>
|
||||
isVoiceMode && activeServerId === serverId && activeAgentId === agentId,
|
||||
[activeAgentId, activeServerId, isVoiceMode]
|
||||
);
|
||||
|
||||
const value: VoiceContextValue = {
|
||||
isVoiceMode,
|
||||
isVoiceSwitching,
|
||||
volume: realtimeAudio.volume,
|
||||
isMuted: realtimeAudio.isMuted,
|
||||
isDetecting: realtimeAudio.isDetecting,
|
||||
isSpeaking: realtimeAudio.isSpeaking,
|
||||
segmentDuration: realtimeAudio.segmentDuration,
|
||||
startVoice,
|
||||
stopVoice,
|
||||
isVoiceModeForAgent,
|
||||
toggleMute: realtimeAudio.toggleMute,
|
||||
activeServerId,
|
||||
activeAgentId,
|
||||
};
|
||||
};
|
||||
}, [providerId, runtime]);
|
||||
|
||||
return (
|
||||
<VoiceContext.Provider value={value}>
|
||||
{children}
|
||||
</VoiceContext.Provider>
|
||||
<VoiceAudioEngineContext.Provider value={engine}>
|
||||
<VoiceRuntimeContext.Provider value={runtime}>
|
||||
{children}
|
||||
</VoiceRuntimeContext.Provider>
|
||||
</VoiceAudioEngineContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user