Update files
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
import { useCallback, useMemo, useState, useEffect } from "react";
|
import { useCallback, useMemo, useState, useEffect } from "react";
|
||||||
import { View, Pressable, Text, Platform, Alert } from "react-native";
|
import { View, Pressable, Text, Platform, Alert, ActivityIndicator } from "react-native";
|
||||||
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
||||||
import Animated, {
|
import Animated, {
|
||||||
useAnimatedStyle,
|
useAnimatedStyle,
|
||||||
@@ -57,7 +57,7 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
const trafficLightPadding = useTrafficLightPadding();
|
const trafficLightPadding = useTrafficLightPadding();
|
||||||
const dragHandlers = useTauriDragHandlers();
|
const dragHandlers = useTauriDragHandlers();
|
||||||
const { connectionStates } = useDaemonConnections();
|
const { connectionStates } = useDaemonConnections();
|
||||||
const { isVoiceMode, startVoice, stopVoice } = useVoice();
|
const { isVoiceMode, isVoiceSwitching, startVoice, stopVoice } = useVoice();
|
||||||
const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false);
|
const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false);
|
||||||
|
|
||||||
// Track user-initiated refresh to avoid showing spinner on background revalidation
|
// Track user-initiated refresh to avoid showing spinner on background revalidation
|
||||||
@@ -148,6 +148,9 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
const hasAnyConfiguredHosts = connectionStates.size > 0;
|
const hasAnyConfiguredHosts = connectionStates.size > 0;
|
||||||
|
|
||||||
const handleToggleVoice = useCallback(() => {
|
const handleToggleVoice = useCallback(() => {
|
||||||
|
if (isVoiceSwitching) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (isVoiceMode) {
|
if (isVoiceMode) {
|
||||||
void stopVoice().catch((error) => {
|
void stopVoice().catch((error) => {
|
||||||
console.error("[SlidingSidebar] Failed to stop voice", error);
|
console.error("[SlidingSidebar] Failed to stop voice", error);
|
||||||
@@ -181,7 +184,7 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
setShowVoiceHostPicker(true);
|
setShowVoiceHostPicker(true);
|
||||||
}, [hasAnyConfiguredHosts, isVoiceMode, startVoice, stopVoice, voiceEligibleHosts]);
|
}, [hasAnyConfiguredHosts, isVoiceMode, isVoiceSwitching, startVoice, stopVoice, voiceEligibleHosts]);
|
||||||
|
|
||||||
const handleSelectVoiceHost = useCallback(
|
const handleSelectVoiceHost = useCallback(
|
||||||
(serverId: string) => {
|
(serverId: string) => {
|
||||||
@@ -324,11 +327,13 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
accessible
|
accessible
|
||||||
accessibilityLabel="Voice"
|
accessibilityLabel="Voice"
|
||||||
accessibilityRole="button"
|
accessibilityRole="button"
|
||||||
|
disabled={isVoiceSwitching}
|
||||||
onPress={handleToggleVoice}
|
onPress={handleToggleVoice}
|
||||||
>
|
>
|
||||||
{({ hovered }) => (
|
{({ hovered }) => (
|
||||||
<AudioLines
|
isVoiceSwitching ? (
|
||||||
size={20}
|
<ActivityIndicator
|
||||||
|
size="small"
|
||||||
color={
|
color={
|
||||||
isVoiceMode
|
isVoiceMode
|
||||||
? theme.colors.foreground
|
? theme.colors.foreground
|
||||||
@@ -337,7 +342,19 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
: theme.colors.foregroundMuted
|
: theme.colors.foregroundMuted
|
||||||
}
|
}
|
||||||
/>
|
/>
|
||||||
)}
|
) : (
|
||||||
|
<AudioLines
|
||||||
|
size={20}
|
||||||
|
color={
|
||||||
|
isVoiceMode
|
||||||
|
? theme.colors.foreground
|
||||||
|
: hovered
|
||||||
|
? theme.colors.foreground
|
||||||
|
: theme.colors.foregroundMuted
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
)
|
||||||
|
)}
|
||||||
</Pressable>
|
</Pressable>
|
||||||
<Pressable
|
<Pressable
|
||||||
style={styles.footerIconButton}
|
style={styles.footerIconButton}
|
||||||
@@ -448,19 +465,33 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
|||||||
accessible
|
accessible
|
||||||
accessibilityLabel="Voice"
|
accessibilityLabel="Voice"
|
||||||
accessibilityRole="button"
|
accessibilityRole="button"
|
||||||
|
disabled={isVoiceSwitching}
|
||||||
onPress={handleToggleVoice}
|
onPress={handleToggleVoice}
|
||||||
>
|
>
|
||||||
{({ hovered }) => (
|
{({ hovered }) => (
|
||||||
<AudioLines
|
isVoiceSwitching ? (
|
||||||
size={20}
|
<ActivityIndicator
|
||||||
color={
|
size="small"
|
||||||
isVoiceMode
|
color={
|
||||||
? theme.colors.foreground
|
isVoiceMode
|
||||||
: hovered
|
|
||||||
? theme.colors.foreground
|
? theme.colors.foreground
|
||||||
: theme.colors.foregroundMuted
|
: hovered
|
||||||
}
|
? theme.colors.foreground
|
||||||
/>
|
: theme.colors.foregroundMuted
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<AudioLines
|
||||||
|
size={20}
|
||||||
|
color={
|
||||||
|
isVoiceMode
|
||||||
|
? theme.colors.foreground
|
||||||
|
: hovered
|
||||||
|
? theme.colors.foreground
|
||||||
|
: theme.colors.foregroundMuted
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
)
|
||||||
)}
|
)}
|
||||||
</Pressable>
|
</Pressable>
|
||||||
<Pressable
|
<Pressable
|
||||||
|
|||||||
@@ -872,10 +872,7 @@ export function SessionProvider({
|
|||||||
const { agentId, event, timestamp } = message.payload;
|
const { agentId, event, timestamp } = message.payload;
|
||||||
const parsedTimestamp = new Date(timestamp);
|
const parsedTimestamp = new Date(timestamp);
|
||||||
|
|
||||||
console.log("[Session] agent_stream", { agentId, event, timestamp });
|
|
||||||
|
|
||||||
if (event.type === "attention_required") {
|
if (event.type === "attention_required") {
|
||||||
console.log("[Session] attention_required", { agentId, shouldNotify: event.shouldNotify, reason: event.reason });
|
|
||||||
if (event.shouldNotify) {
|
if (event.shouldNotify) {
|
||||||
notifyAgentAttention({
|
notifyAgentAttention({
|
||||||
agentId,
|
agentId,
|
||||||
@@ -926,11 +923,6 @@ export function SessionProvider({
|
|||||||
if (message.type !== "agent_stream_snapshot") return;
|
if (message.type !== "agent_stream_snapshot") return;
|
||||||
const { agentId, events } = message.payload;
|
const { agentId, events } = message.payload;
|
||||||
|
|
||||||
console.log("[Session] agent_stream_snapshot", {
|
|
||||||
agentId,
|
|
||||||
eventCount: events.length,
|
|
||||||
});
|
|
||||||
|
|
||||||
const hydrated = hydrateStreamState(
|
const hydrated = hydrateStreamState(
|
||||||
events.map(({ event, timestamp }) => ({
|
events.map(({ event, timestamp }) => ({
|
||||||
event: event as AgentStreamEventPayload,
|
event: event as AgentStreamEventPayload,
|
||||||
@@ -1059,15 +1051,8 @@ export function SessionProvider({
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (!isFinalChunk) {
|
if (!isFinalChunk) {
|
||||||
console.log(
|
|
||||||
`[Session] Buffered chunk ${chunkIndex} for group ${playbackGroupId}`
|
|
||||||
);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(
|
|
||||||
`[Session] Received final chunk for group ${playbackGroupId}, total chunks: ${buffer.length}`
|
|
||||||
);
|
|
||||||
buffer.sort((a, b) => a.chunkIndex - b.chunkIndex);
|
buffer.sort((a, b) => a.chunkIndex - b.chunkIndex);
|
||||||
|
|
||||||
let playbackFailed = false;
|
let playbackFailed = false;
|
||||||
@@ -1109,10 +1094,6 @@ export function SessionProvider({
|
|||||||
offset += chunk.length;
|
offset += chunk.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(
|
|
||||||
`[Session] Playing concatenated audio: ${buffer.length} chunks, ${totalSize} bytes`
|
|
||||||
);
|
|
||||||
|
|
||||||
const audioBlob = {
|
const audioBlob = {
|
||||||
type: mimeType,
|
type: mimeType,
|
||||||
size: totalSize,
|
size: totalSize,
|
||||||
@@ -1264,11 +1245,7 @@ export function SessionProvider({
|
|||||||
const transcriptText = message.payload.text.trim();
|
const transcriptText = message.payload.text.trim();
|
||||||
|
|
||||||
if (!transcriptText) {
|
if (!transcriptText) {
|
||||||
console.log(
|
|
||||||
"[Session] Empty transcription (false positive) - ignoring"
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
console.log("[Session] Transcription received - stopping playback");
|
|
||||||
audioPlayer.stop();
|
audioPlayer.stop();
|
||||||
setIsPlayingAudio(serverId, false);
|
setIsPlayingAudio(serverId, false);
|
||||||
setCurrentAssistantMessage(serverId, "");
|
setCurrentAssistantMessage(serverId, "");
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ const VOICE_VAD_DETECTION_GRACE_PERIOD_MS = 700;
|
|||||||
|
|
||||||
interface VoiceContextValue {
|
interface VoiceContextValue {
|
||||||
isVoiceMode: boolean;
|
isVoiceMode: boolean;
|
||||||
|
isVoiceSwitching: boolean;
|
||||||
volume: number;
|
volume: number;
|
||||||
isMuted: boolean;
|
isMuted: boolean;
|
||||||
isDetecting: boolean;
|
isDetecting: boolean;
|
||||||
@@ -57,7 +58,12 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
);
|
);
|
||||||
const realtimeSessionRef = useRef<SessionState | null>(null);
|
const realtimeSessionRef = useRef<SessionState | null>(null);
|
||||||
const [isVoiceMode, setIsVoiceMode] = useState(false);
|
const [isVoiceMode, setIsVoiceMode] = useState(false);
|
||||||
|
const [isVoiceSwitching, setIsVoiceSwitching] = useState(false);
|
||||||
const bargeInPlaybackStopRef = useRef<number | null>(null);
|
const bargeInPlaybackStopRef = useRef<number | null>(null);
|
||||||
|
const wasVoiceSocketConnectedRef = useRef(false);
|
||||||
|
const lastVoiceModeSyncedClientRef = useRef<SessionState["client"] | null>(null);
|
||||||
|
const voiceTransportReadyRef = useRef(false);
|
||||||
|
const voiceResyncInFlightRef = useRef(false);
|
||||||
|
|
||||||
const realtimeAudio = useSpeechmaticsAudio({
|
const realtimeAudio = useSpeechmaticsAudio({
|
||||||
onSpeechStart: () => {
|
onSpeechStart: () => {
|
||||||
@@ -91,6 +97,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
console.log("[Voice] Speech ended");
|
console.log("[Voice] Speech ended");
|
||||||
},
|
},
|
||||||
onAudioSegment: ({ audioData, isLast }) => {
|
onAudioSegment: ({ audioData, isLast }) => {
|
||||||
|
if (!voiceTransportReadyRef.current) {
|
||||||
|
console.log("[Voice] Skipping audio segment: voice transport not ready");
|
||||||
|
return;
|
||||||
|
}
|
||||||
console.log(
|
console.log(
|
||||||
"[Voice] Sending audio segment, length:",
|
"[Voice] Sending audio segment, length:",
|
||||||
audioData.length,
|
audioData.length,
|
||||||
@@ -134,6 +144,48 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
realtimeSessionRef.current = activeSession;
|
realtimeSessionRef.current = activeSession;
|
||||||
}, [activeSession]);
|
}, [activeSession]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const connected = activeSession?.connection.isConnected ?? false;
|
||||||
|
const client = activeSession?.client ?? null;
|
||||||
|
if (!connected) {
|
||||||
|
voiceTransportReadyRef.current = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isVoiceMode || !activeServerId || !client) {
|
||||||
|
wasVoiceSocketConnectedRef.current = connected;
|
||||||
|
if (!isVoiceMode) {
|
||||||
|
lastVoiceModeSyncedClientRef.current = null;
|
||||||
|
}
|
||||||
|
voiceTransportReadyRef.current = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const connectionRecovered = connected && !wasVoiceSocketConnectedRef.current;
|
||||||
|
const clientChanged = lastVoiceModeSyncedClientRef.current !== client;
|
||||||
|
if (connected && (connectionRecovered || clientChanged)) {
|
||||||
|
if (!voiceResyncInFlightRef.current) {
|
||||||
|
voiceResyncInFlightRef.current = true;
|
||||||
|
voiceTransportReadyRef.current = false;
|
||||||
|
setIsVoiceSwitching(true);
|
||||||
|
void client.setVoiceMode(true).then(
|
||||||
|
() => {
|
||||||
|
console.log("[Voice] Re-synced voice mode after reconnect");
|
||||||
|
lastVoiceModeSyncedClientRef.current = client;
|
||||||
|
voiceTransportReadyRef.current = true;
|
||||||
|
},
|
||||||
|
(error) => {
|
||||||
|
console.error("[Voice] Failed to re-sync voice mode:", error);
|
||||||
|
}
|
||||||
|
).finally(() => {
|
||||||
|
voiceResyncInFlightRef.current = false;
|
||||||
|
setIsVoiceSwitching(false);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
wasVoiceSocketConnectedRef.current = connected;
|
||||||
|
}, [activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]);
|
||||||
|
|
||||||
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
|
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -155,56 +207,67 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
throw new Error(`Host ${serverId} is not connected`);
|
throw new Error(`Host ${serverId} is not connected`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setIsVoiceSwitching(true);
|
||||||
|
voiceTransportReadyRef.current = false;
|
||||||
try {
|
try {
|
||||||
realtimeSessionRef.current = session;
|
realtimeSessionRef.current = session;
|
||||||
setActiveServerId(serverId);
|
setActiveServerId(serverId);
|
||||||
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
|
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
|
||||||
console.warn("[Voice] Failed to activate keep-awake:", error);
|
console.warn("[Voice] Failed to activate keep-awake:", error);
|
||||||
});
|
});
|
||||||
await session.audioPlayer?.warmup?.();
|
|
||||||
await realtimeAudio.start();
|
|
||||||
setIsVoiceMode(true);
|
|
||||||
console.log("[Voice] Mode enabled");
|
|
||||||
|
|
||||||
if (session?.client) {
|
if (session?.client) {
|
||||||
await session.client.setVoiceMode(true);
|
await session.client.setVoiceMode(true);
|
||||||
} else {
|
} else {
|
||||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||||
}
|
}
|
||||||
|
await session.audioPlayer?.warmup?.();
|
||||||
|
await realtimeAudio.start();
|
||||||
|
voiceTransportReadyRef.current = true;
|
||||||
|
setIsVoiceMode(true);
|
||||||
|
lastVoiceModeSyncedClientRef.current = session.client;
|
||||||
|
console.log("[Voice] Mode enabled");
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
console.error("[Voice] Failed to start:", error);
|
console.error("[Voice] Failed to start:", error);
|
||||||
|
await realtimeAudio.stop().catch(() => undefined);
|
||||||
setActiveServerId((current) => (current === serverId ? null : current));
|
setActiveServerId((current) => (current === serverId ? null : current));
|
||||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||||
throw error;
|
throw error;
|
||||||
|
} finally {
|
||||||
|
setIsVoiceSwitching(false);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[getSession, realtimeAudio]
|
[getSession, realtimeAudio]
|
||||||
);
|
);
|
||||||
|
|
||||||
const stopVoice = useCallback(async () => {
|
const stopVoice = useCallback(async () => {
|
||||||
|
setIsVoiceSwitching(true);
|
||||||
|
voiceTransportReadyRef.current = false;
|
||||||
try {
|
try {
|
||||||
const session = realtimeSessionRef.current;
|
const session = realtimeSessionRef.current;
|
||||||
session?.audioPlayer?.stop();
|
session?.audioPlayer?.stop();
|
||||||
|
if (session?.client) {
|
||||||
|
await session.client.setVoiceMode(false);
|
||||||
|
lastVoiceModeSyncedClientRef.current = session.client;
|
||||||
|
} else {
|
||||||
|
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||||
|
}
|
||||||
await realtimeAudio.stop();
|
await realtimeAudio.stop();
|
||||||
setIsVoiceMode(false);
|
setIsVoiceMode(false);
|
||||||
setActiveServerId(null);
|
setActiveServerId(null);
|
||||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||||
console.log("[Voice] Mode disabled");
|
console.log("[Voice] Mode disabled");
|
||||||
|
|
||||||
if (session?.client) {
|
|
||||||
await session.client.setVoiceMode(false);
|
|
||||||
} else {
|
|
||||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
console.error("[Voice] Failed to stop:", error);
|
console.error("[Voice] Failed to stop:", error);
|
||||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||||
throw error;
|
throw error;
|
||||||
|
} finally {
|
||||||
|
setIsVoiceSwitching(false);
|
||||||
}
|
}
|
||||||
}, [realtimeAudio]);
|
}, [realtimeAudio]);
|
||||||
|
|
||||||
const value: VoiceContextValue = {
|
const value: VoiceContextValue = {
|
||||||
isVoiceMode,
|
isVoiceMode,
|
||||||
|
isVoiceSwitching,
|
||||||
volume: realtimeAudio.volume,
|
volume: realtimeAudio.volume,
|
||||||
isMuted: realtimeAudio.isMuted,
|
isMuted: realtimeAudio.isMuted,
|
||||||
isDetecting: realtimeAudio.isDetecting,
|
isDetecting: realtimeAudio.isDetecting,
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import { useDaemonConnections } from "@/contexts/daemon-connections-context";
|
|||||||
import { useSessionStore } from "@/stores/session-store";
|
import { useSessionStore } from "@/stores/session-store";
|
||||||
import type { AgentDirectoryEntry } from "@/types/agent-directory";
|
import type { AgentDirectoryEntry } from "@/types/agent-directory";
|
||||||
import type { Agent } from "@/stores/session-store";
|
import type { Agent } from "@/stores/session-store";
|
||||||
import { isPerfLoggingEnabled } from "@/utils/perf";
|
|
||||||
import { derivePendingPermissionKey, normalizeAgentSnapshot } from "@/utils/agent-snapshots";
|
import { derivePendingPermissionKey, normalizeAgentSnapshot } from "@/utils/agent-snapshots";
|
||||||
|
|
||||||
export interface AggregatedAgent extends AgentDirectoryEntry {
|
export interface AggregatedAgent extends AgentDirectoryEntry {
|
||||||
@@ -44,14 +43,12 @@ export function useAggregatedAgents(): AggregatedAgentsResult {
|
|||||||
);
|
);
|
||||||
|
|
||||||
const refreshAll = useCallback(() => {
|
const refreshAll = useCallback(() => {
|
||||||
console.log('[useAggregatedAgents] Manual refresh triggered for all sessions');
|
|
||||||
for (const [serverId, client] of Object.entries(sessionClients)) {
|
for (const [serverId, client] of Object.entries(sessionClients)) {
|
||||||
if (!client) {
|
if (!client) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
void (async () => {
|
void (async () => {
|
||||||
try {
|
try {
|
||||||
console.log(`[useAggregatedAgents] Refreshing session ${serverId}`);
|
|
||||||
const agentsList = await client.fetchAgents({
|
const agentsList = await client.fetchAgents({
|
||||||
filter: { labels: { ui: "true" } },
|
filter: { labels: { ui: "true" } },
|
||||||
});
|
});
|
||||||
@@ -131,27 +128,20 @@ export function useAggregatedAgents(): AggregatedAgentsResult {
|
|||||||
const hasAnyData = allAgents.length > 0;
|
const hasAnyData = allAgents.length > 0;
|
||||||
|
|
||||||
// Check if any connection is currently loading
|
// Check if any connection is currently loading
|
||||||
const connectingReasons: string[] = [];
|
const isConnecting = Array.from(connectionStates.entries()).some(([, c]) => {
|
||||||
const isConnecting = Array.from(connectionStates.entries()).some(([id, c]) => {
|
|
||||||
const shortId = id.substring(0, 20);
|
|
||||||
|
|
||||||
// First-time connection (never received agent list)
|
// First-time connection (never received agent list)
|
||||||
if (c.status === 'connecting' && !c.hasEverReceivedAgentList) {
|
if (c.status === 'connecting' && !c.hasEverReceivedAgentList) {
|
||||||
connectingReasons.push(`${shortId}: first-time connecting`);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (c.status === 'online' && !c.hasEverReceivedAgentList) {
|
if (c.status === 'online' && !c.hasEverReceivedAgentList) {
|
||||||
connectingReasons.push(`${shortId}: online but no fetch_agents yet`);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reconnecting (have received agent list before)
|
// Reconnecting (have received agent list before)
|
||||||
if (c.status === 'connecting' && c.hasEverReceivedAgentList) {
|
if (c.status === 'connecting' && c.hasEverReceivedAgentList) {
|
||||||
connectingReasons.push(`${shortId}: reconnecting`);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (c.status === 'online' && !c.agentListReady && c.hasEverReceivedAgentList) {
|
if (c.status === 'online' && !c.agentListReady && c.hasEverReceivedAgentList) {
|
||||||
connectingReasons.push(`${shortId}: online but agentListReady=false (waiting for fetch_agents)`);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,28 +157,6 @@ export function useAggregatedAgents(): AggregatedAgentsResult {
|
|||||||
// isLoading: Generic loading flag (either initial or revalidating)
|
// isLoading: Generic loading flag (either initial or revalidating)
|
||||||
const isLoading = isConnecting;
|
const isLoading = isConnecting;
|
||||||
|
|
||||||
if (isPerfLoggingEnabled()) {
|
|
||||||
const connectionStatesArray = Array.from(connectionStates.entries()).map(([id, state]) => ({
|
|
||||||
id: id.substring(0, 20) + (id.length > 20 ? '...' : ''),
|
|
||||||
status: state.status,
|
|
||||||
agentListReady: state.agentListReady,
|
|
||||||
hasEverReceivedAgentList: state.hasEverReceivedAgentList,
|
|
||||||
}));
|
|
||||||
|
|
||||||
console.log('[useAggregatedAgents] States:', {
|
|
||||||
hasAnyData,
|
|
||||||
isConnecting,
|
|
||||||
isInitialLoad,
|
|
||||||
isRevalidating,
|
|
||||||
totalConnectionStates: connectionStates.size,
|
|
||||||
connectingReasons: connectingReasons.length > 0 ? connectingReasons : 'none',
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log('[useAggregatedAgents] Connection States Detail:',
|
|
||||||
JSON.stringify(connectionStatesArray, null, 2)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
agents: allAgents,
|
agents: allAgents,
|
||||||
isLoading,
|
isLoading,
|
||||||
|
|||||||
@@ -210,9 +210,6 @@ export interface GroupedAgents {
|
|||||||
|
|
||||||
const ACTIVE_GRACE_PERIOD_MS = 2 * 24 * 60 * 60 * 1000; // 2 days (temporary for screenshots)
|
const ACTIVE_GRACE_PERIOD_MS = 2 * 24 * 60 * 60 * 1000; // 2 days (temporary for screenshots)
|
||||||
|
|
||||||
// eslint-disable-next-line no-console
|
|
||||||
console.log('[agent-grouping] ACTIVE_GRACE_PERIOD_MS:', ACTIVE_GRACE_PERIOD_MS, 'ms =', ACTIVE_GRACE_PERIOD_MS / (1000 * 60 * 60), 'hours');
|
|
||||||
|
|
||||||
interface GroupAgentsOptions {
|
interface GroupAgentsOptions {
|
||||||
/**
|
/**
|
||||||
* Optional function to read a remote URL for an agent.
|
* Optional function to read a remote URL for an agent.
|
||||||
@@ -252,17 +249,6 @@ export function groupAgents(
|
|||||||
const isRecentlyActive = ageDiff < ACTIVE_GRACE_PERIOD_MS;
|
const isRecentlyActive = ageDiff < ACTIVE_GRACE_PERIOD_MS;
|
||||||
const isActive = isRunningOrAttention || isRecentlyActive;
|
const isActive = isRunningOrAttention || isRecentlyActive;
|
||||||
|
|
||||||
// eslint-disable-next-line no-console
|
|
||||||
console.log('[agent-grouping] agent:', agent.title || agent.id, {
|
|
||||||
status: agent.status,
|
|
||||||
lastActivityAt: agent.lastActivityAt,
|
|
||||||
ageDiffHours: ageDiff / (1000 * 60 * 60),
|
|
||||||
gracePeriodHours: ACTIVE_GRACE_PERIOD_MS / (1000 * 60 * 60),
|
|
||||||
isRunningOrAttention,
|
|
||||||
isRecentlyActive,
|
|
||||||
isActive,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (isActive) {
|
if (isActive) {
|
||||||
activeAgents.push(agent);
|
activeAgents.push(agent);
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -1,21 +1,4 @@
|
|||||||
const getIsDevEnvironment = (): boolean => {
|
const PERF_LOGGING_ENABLED = false;
|
||||||
const globalDev = (globalThis as { __DEV__?: boolean } | undefined)?.__DEV__;
|
|
||||||
if (typeof globalDev === "boolean") {
|
|
||||||
return globalDev;
|
|
||||||
}
|
|
||||||
if (typeof process !== "undefined" && process.env?.NODE_ENV) {
|
|
||||||
return process.env.NODE_ENV !== "production";
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
const shouldDisablePerfLogging =
|
|
||||||
typeof process !== "undefined" && process.env?.EXPO_PUBLIC_DISABLE_PERF_LOGGING === "1";
|
|
||||||
const shouldForcePerfLogging =
|
|
||||||
typeof process !== "undefined" && process.env?.EXPO_PUBLIC_ENABLE_PERF_LOGGING === "1";
|
|
||||||
|
|
||||||
const PERF_LOGGING_ENABLED =
|
|
||||||
(shouldForcePerfLogging || getIsDevEnvironment()) && !shouldDisablePerfLogging;
|
|
||||||
|
|
||||||
export const isPerfLoggingEnabled = (): boolean => PERF_LOGGING_ENABLED;
|
export const isPerfLoggingEnabled = (): boolean => PERF_LOGGING_ENABLED;
|
||||||
|
|
||||||
|
|||||||
@@ -201,6 +201,10 @@ type SpeechModelsListPayload = SpeechModelsListResponse["payload"];
|
|||||||
type SpeechModelsDownloadPayload = SpeechModelsDownloadResponse["payload"];
|
type SpeechModelsDownloadPayload = SpeechModelsDownloadResponse["payload"];
|
||||||
type ListCommandsPayload = ListCommandsResponse["payload"];
|
type ListCommandsPayload = ListCommandsResponse["payload"];
|
||||||
type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
|
type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
|
||||||
|
type SetVoiceModePayload = Extract<
|
||||||
|
SessionOutboundMessage,
|
||||||
|
{ type: "set_voice_mode_response" }
|
||||||
|
>["payload"];
|
||||||
type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
|
type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
|
||||||
type ListTerminalsPayload = ListTerminalsResponse["payload"];
|
type ListTerminalsPayload = ListTerminalsResponse["payload"];
|
||||||
type CreateTerminalPayload = CreateTerminalResponse["payload"];
|
type CreateTerminalPayload = CreateTerminalResponse["payload"];
|
||||||
@@ -1285,8 +1289,31 @@ export class DaemonClient {
|
|||||||
// Audio / Voice
|
// Audio / Voice
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise<void> {
|
async setVoiceMode(
|
||||||
this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId });
|
enabled: boolean,
|
||||||
|
voiceAgentId?: string
|
||||||
|
): Promise<SetVoiceModePayload> {
|
||||||
|
const requestId = this.createRequestId();
|
||||||
|
const message = SessionInboundMessageSchema.parse({
|
||||||
|
type: "set_voice_mode",
|
||||||
|
enabled,
|
||||||
|
...(voiceAgentId ? { voiceAgentId } : {}),
|
||||||
|
requestId,
|
||||||
|
});
|
||||||
|
return this.sendRequest({
|
||||||
|
requestId,
|
||||||
|
message,
|
||||||
|
timeout: 10000,
|
||||||
|
select: (msg) => {
|
||||||
|
if (msg.type !== "set_voice_mode_response") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (msg.payload.requestId !== requestId) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return msg.payload;
|
||||||
|
},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async sendVoiceAudioChunk(
|
async sendVoiceAudioChunk(
|
||||||
|
|||||||
@@ -43,5 +43,83 @@ describe("TTSManager", () => {
|
|||||||
expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1);
|
expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1);
|
||||||
expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true);
|
expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
|
||||||
|
|
||||||
|
it("splits long text into safe synthesis segments", async () => {
|
||||||
|
const calls: string[] = [];
|
||||||
|
const tts: TextToSpeechProvider = {
|
||||||
|
async synthesizeSpeech(text: string): Promise<{ stream: Readable; format: string }> {
|
||||||
|
calls.push(text);
|
||||||
|
return {
|
||||||
|
stream: Readable.from([Buffer.from("x")]),
|
||||||
|
format: "pcm;rate=24000",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const manager = new TTSManager("s1", pino({ level: "silent" }), tts);
|
||||||
|
const abort = new AbortController();
|
||||||
|
const longText = Array.from({ length: 180 })
|
||||||
|
.map((_, i) => `Sentence ${i + 1}.`)
|
||||||
|
.join(" ");
|
||||||
|
|
||||||
|
await manager.generateAndWaitForPlayback(
|
||||||
|
longText,
|
||||||
|
(msg) => {
|
||||||
|
if (msg.type === "audio_output") {
|
||||||
|
manager.confirmAudioPlayed(msg.payload.id);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
abort.signal,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(calls.length).toBeGreaterThan(1);
|
||||||
|
expect(calls.every((text) => text.length <= 400)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not emit unhandled rejections when stream iteration fails", async () => {
|
||||||
|
const unhandled: unknown[] = [];
|
||||||
|
const onUnhandled = (reason: unknown) => {
|
||||||
|
unhandled.push(reason);
|
||||||
|
};
|
||||||
|
process.on("unhandledRejection", onUnhandled);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const tts: TextToSpeechProvider = {
|
||||||
|
async synthesizeSpeech(): Promise<{ stream: Readable; format: string }> {
|
||||||
|
const stream = Readable.from(
|
||||||
|
(async function* () {
|
||||||
|
yield Buffer.from("a");
|
||||||
|
throw new Error("stream exploded");
|
||||||
|
})()
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
stream,
|
||||||
|
format: "pcm;rate=24000",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const manager = new TTSManager("s1", pino({ level: "silent" }), tts);
|
||||||
|
const abort = new AbortController();
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
manager.generateAndWaitForPlayback(
|
||||||
|
"hello",
|
||||||
|
(msg) => {
|
||||||
|
if (msg.type === "audio_output") {
|
||||||
|
manager.confirmAudioPlayed(msg.payload.id);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
abort.signal,
|
||||||
|
true
|
||||||
|
)
|
||||||
|
).rejects.toThrow("stream exploded");
|
||||||
|
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
expect(unhandled).toHaveLength(0);
|
||||||
|
} finally {
|
||||||
|
process.off("unhandledRejection", onUnhandled);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -10,6 +10,84 @@ interface PendingPlayback {
|
|||||||
streamEnded: boolean;
|
streamEnded: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const MAX_TTS_SEGMENT_CHARS = 400;
|
||||||
|
|
||||||
|
function splitTextForTts(text: string, maxChars: number): string[] {
|
||||||
|
const normalized = text.trim().replace(/\s+/g, " ");
|
||||||
|
if (!normalized) {
|
||||||
|
throw new Error("Cannot synthesize empty text");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized.length <= maxChars) {
|
||||||
|
return [normalized];
|
||||||
|
}
|
||||||
|
|
||||||
|
const parts: string[] = [];
|
||||||
|
const sentenceChunks = normalized.split(/(?<=[.!?])\s+/);
|
||||||
|
|
||||||
|
let current = "";
|
||||||
|
const pushCurrent = () => {
|
||||||
|
const trimmed = current.trim();
|
||||||
|
if (trimmed.length > 0) {
|
||||||
|
parts.push(trimmed);
|
||||||
|
}
|
||||||
|
current = "";
|
||||||
|
};
|
||||||
|
|
||||||
|
const appendFragment = (fragment: string) => {
|
||||||
|
const trimmed = fragment.trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!current) {
|
||||||
|
current = trimmed;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidate = `${current} ${trimmed}`;
|
||||||
|
if (candidate.length <= maxChars) {
|
||||||
|
current = candidate;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
pushCurrent();
|
||||||
|
current = trimmed;
|
||||||
|
};
|
||||||
|
|
||||||
|
const splitLargeFragment = (fragment: string): string[] => {
|
||||||
|
const trimmed = fragment.trim();
|
||||||
|
if (trimmed.length <= maxChars) {
|
||||||
|
return [trimmed];
|
||||||
|
}
|
||||||
|
|
||||||
|
const out: string[] = [];
|
||||||
|
let remaining = trimmed;
|
||||||
|
while (remaining.length > maxChars) {
|
||||||
|
let idx = remaining.lastIndexOf(" ", maxChars);
|
||||||
|
if (idx < Math.floor(maxChars * 0.5)) {
|
||||||
|
idx = maxChars;
|
||||||
|
}
|
||||||
|
out.push(remaining.slice(0, idx).trim());
|
||||||
|
remaining = remaining.slice(idx).trim();
|
||||||
|
}
|
||||||
|
if (remaining.length > 0) {
|
||||||
|
out.push(remaining);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const sentence of sentenceChunks) {
|
||||||
|
const fragments = splitLargeFragment(sentence);
|
||||||
|
for (const fragment of fragments) {
|
||||||
|
appendFragment(fragment);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pushCurrent();
|
||||||
|
|
||||||
|
return parts;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Per-session TTS manager
|
* Per-session TTS manager
|
||||||
* Handles TTS audio generation and playback confirmation tracking
|
* Handles TTS audio generation and playback confirmation tracking
|
||||||
@@ -33,6 +111,37 @@ export class TTSManager {
|
|||||||
emitMessage: (msg: SessionOutboundMessage) => void,
|
emitMessage: (msg: SessionOutboundMessage) => void,
|
||||||
abortSignal: AbortSignal,
|
abortSignal: AbortSignal,
|
||||||
isVoiceMode: boolean
|
isVoiceMode: boolean
|
||||||
|
): Promise<void> {
|
||||||
|
this.logger.info(
|
||||||
|
{
|
||||||
|
isVoiceMode,
|
||||||
|
textLength: text.length,
|
||||||
|
text,
|
||||||
|
},
|
||||||
|
"TTS input text"
|
||||||
|
);
|
||||||
|
|
||||||
|
const segments = splitTextForTts(text, MAX_TTS_SEGMENT_CHARS);
|
||||||
|
for (const segment of segments) {
|
||||||
|
if (abortSignal.aborted) {
|
||||||
|
this.logger.debug("Aborted before generating segmented audio");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.generateSegmentAndWaitForPlayback(
|
||||||
|
segment,
|
||||||
|
emitMessage,
|
||||||
|
abortSignal,
|
||||||
|
isVoiceMode
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async generateSegmentAndWaitForPlayback(
|
||||||
|
text: string,
|
||||||
|
emitMessage: (msg: SessionOutboundMessage) => void,
|
||||||
|
abortSignal: AbortSignal,
|
||||||
|
isVoiceMode: boolean
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (!this.tts) {
|
if (!this.tts) {
|
||||||
throw new Error("TTS not configured");
|
throw new Error("TTS not configured");
|
||||||
@@ -121,11 +230,6 @@ export class TTSManager {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
this.logger.debug(
|
|
||||||
{ chunkId, isLastChunk: next.done },
|
|
||||||
"Sent audio chunk"
|
|
||||||
);
|
|
||||||
|
|
||||||
chunkIndex += 1;
|
chunkIndex += 1;
|
||||||
|
|
||||||
if (next.done) {
|
if (next.done) {
|
||||||
@@ -151,7 +255,6 @@ export class TTSManager {
|
|||||||
} else {
|
} else {
|
||||||
this.logger.error({ err: error }, "Error streaming audio");
|
this.logger.error({ err: error }, "Error streaming audio");
|
||||||
this.pendingPlaybacks.delete(audioId);
|
this.pendingPlaybacks.delete(audioId);
|
||||||
pendingPlayback.reject(error as Error);
|
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -147,8 +147,16 @@ describe("daemon client E2E", () => {
|
|||||||
});
|
});
|
||||||
expect(Array.isArray(voiceAgents)).toBe(true);
|
expect(Array.isArray(voiceAgents)).toBe(true);
|
||||||
|
|
||||||
await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined();
|
await expect(ctx.client.setVoiceMode(true)).resolves.toMatchObject({
|
||||||
await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined();
|
enabled: true,
|
||||||
|
accepted: true,
|
||||||
|
error: null,
|
||||||
|
});
|
||||||
|
await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({
|
||||||
|
enabled: false,
|
||||||
|
accepted: true,
|
||||||
|
error: null,
|
||||||
|
});
|
||||||
|
|
||||||
await ctx.client.deleteAgent(randomUUID());
|
await ctx.client.deleteAgent(randomUUID());
|
||||||
}, 30000);
|
}, 30000);
|
||||||
|
|||||||
@@ -879,7 +879,7 @@ export class Session {
|
|||||||
break;
|
break;
|
||||||
|
|
||||||
case "set_voice_mode":
|
case "set_voice_mode":
|
||||||
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId);
|
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId, msg.requestId);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case "send_agent_message_request":
|
case "send_agent_message_request":
|
||||||
@@ -1259,7 +1259,8 @@ export class Session {
|
|||||||
*/
|
*/
|
||||||
private async handleSetVoiceMode(
|
private async handleSetVoiceMode(
|
||||||
enabled: boolean,
|
enabled: boolean,
|
||||||
voiceAgentId?: string
|
voiceAgentId?: string,
|
||||||
|
requestId?: string
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (enabled) {
|
if (enabled) {
|
||||||
let normalizedVoiceAgentId: string | null = null;
|
let normalizedVoiceAgentId: string | null = null;
|
||||||
@@ -1282,6 +1283,18 @@ export class Session {
|
|||||||
},
|
},
|
||||||
"Voice mode enabled (agent-backed)"
|
"Voice mode enabled (agent-backed)"
|
||||||
);
|
);
|
||||||
|
if (requestId) {
|
||||||
|
this.emit({
|
||||||
|
type: "set_voice_mode_response",
|
||||||
|
payload: {
|
||||||
|
requestId,
|
||||||
|
enabled: true,
|
||||||
|
voiceAgentId: this.voiceAssistantAgentId,
|
||||||
|
accepted: true,
|
||||||
|
error: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1290,6 +1303,18 @@ export class Session {
|
|||||||
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
|
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
|
||||||
"Voice mode disabled (agent-backed)"
|
"Voice mode disabled (agent-backed)"
|
||||||
);
|
);
|
||||||
|
if (requestId) {
|
||||||
|
this.emit({
|
||||||
|
type: "set_voice_mode_response",
|
||||||
|
payload: {
|
||||||
|
requestId,
|
||||||
|
enabled: false,
|
||||||
|
voiceAgentId: this.voiceAssistantAgentId,
|
||||||
|
accepted: true,
|
||||||
|
error: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private parseVoiceAgentId(rawId: string, source: string): string {
|
private parseVoiceAgentId(rawId: string, source: string): string {
|
||||||
@@ -2471,7 +2496,6 @@ export class Session {
|
|||||||
lastActivityAt: string;
|
lastActivityAt: string;
|
||||||
appVisible: boolean;
|
appVisible: boolean;
|
||||||
}): void {
|
}): void {
|
||||||
this.sessionLogger.debug({ heartbeat: msg }, "Client heartbeat");
|
|
||||||
this.clientActivity = {
|
this.clientActivity = {
|
||||||
deviceType: msg.deviceType,
|
deviceType: msg.deviceType,
|
||||||
focusedAgentId: msg.focusedAgentId,
|
focusedAgentId: msg.focusedAgentId,
|
||||||
@@ -4152,6 +4176,12 @@ export class Session {
|
|||||||
private async handleAudioChunk(
|
private async handleAudioChunk(
|
||||||
msg: Extract<SessionInboundMessage, { type: "voice_audio_chunk" }>
|
msg: Extract<SessionInboundMessage, { type: "voice_audio_chunk" }>
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
if (!this.isVoiceMode) {
|
||||||
|
this.sessionLogger.warn(
|
||||||
|
"Received voice_audio_chunk while voice mode is disabled; transcript will be emitted but voice assistant turn is skipped"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
await this.handleVoiceSpeechStart();
|
await this.handleVoiceSpeechStart();
|
||||||
|
|
||||||
const chunkBuffer = Buffer.from(msg.audio, "base64");
|
const chunkBuffer = Buffer.from(msg.audio, "base64");
|
||||||
@@ -4193,11 +4223,6 @@ export class Session {
|
|||||||
this.audioBuffer.totalPCMBytes += chunkBuffer.length;
|
this.audioBuffer.totalPCMBytes += chunkBuffer.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.sessionLogger.debug(
|
|
||||||
{ bytes: chunkBuffer.length, chunks: this.audioBuffer.chunks.length, pcmBytes: this.audioBuffer.totalPCMBytes },
|
|
||||||
`Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})`
|
|
||||||
);
|
|
||||||
|
|
||||||
// In voice mode, only process audio when the user has finished speaking (isLast = true)
|
// In voice mode, only process audio when the user has finished speaking (isLast = true)
|
||||||
// This prevents partial transcriptions from being sent to the LLM
|
// This prevents partial transcriptions from being sent to the LLM
|
||||||
if (this.isVoiceMode) {
|
if (this.isVoiceMode) {
|
||||||
@@ -4343,6 +4368,15 @@ export class Session {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const transcriptText = result.text.trim();
|
const transcriptText = result.text.trim();
|
||||||
|
this.sessionLogger.info(
|
||||||
|
{
|
||||||
|
requestId,
|
||||||
|
isVoiceMode: this.isVoiceMode,
|
||||||
|
transcriptLength: transcriptText.length,
|
||||||
|
transcript: transcriptText,
|
||||||
|
},
|
||||||
|
"Transcription result"
|
||||||
|
);
|
||||||
|
|
||||||
// Emit transcription result
|
// Emit transcription result
|
||||||
this.emit({
|
this.emit({
|
||||||
@@ -4408,8 +4442,17 @@ export class Session {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// Set phase to LLM and process (TTS enabled in voice mode for voice agents)
|
|
||||||
this.clearSpeechInProgress("transcription complete");
|
this.clearSpeechInProgress("transcription complete");
|
||||||
|
if (!this.isVoiceMode) {
|
||||||
|
this.sessionLogger.debug(
|
||||||
|
{ requestId },
|
||||||
|
"Skipping voice agent processing because voice mode is disabled"
|
||||||
|
);
|
||||||
|
this.setPhase("idle");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set phase to LLM and process (TTS enabled in voice mode for voice agents)
|
||||||
this.setPhase("llm");
|
this.setPhase("llm");
|
||||||
this.currentStreamPromise = this.processVoiceTurn(result.text);
|
this.currentStreamPromise = this.processVoiceTurn(result.text);
|
||||||
await this.currentStreamPromise;
|
await this.currentStreamPromise;
|
||||||
@@ -4560,7 +4603,6 @@ export class Session {
|
|||||||
this.agentManager.recordUserMessage(agentId, userText);
|
this.agentManager.recordUserMessage(agentId, userText);
|
||||||
|
|
||||||
let sawSpeakToolCall = false;
|
let sawSpeakToolCall = false;
|
||||||
const assistantTextChunks: string[] = [];
|
|
||||||
const iterator = this.agentManager.streamAgent(agentId, prompt);
|
const iterator = this.agentManager.streamAgent(agentId, prompt);
|
||||||
for await (const event of iterator) {
|
for await (const event of iterator) {
|
||||||
if (event.type === "turn_failed") {
|
if (event.type === "turn_failed") {
|
||||||
@@ -4572,9 +4614,6 @@ export class Session {
|
|||||||
sawSpeakToolCall = true;
|
sawSpeakToolCall = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (event.item.type === "assistant_message" && event.item.text.trim().length > 0) {
|
|
||||||
assistantTextChunks.push(event.item.text.trim());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (event.type === "permission_requested") {
|
if (event.type === "permission_requested") {
|
||||||
if (this.shouldAllowVoicePermission(event.request)) {
|
if (this.shouldAllowVoicePermission(event.request)) {
|
||||||
@@ -4594,26 +4633,10 @@ export class Session {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!sawSpeakToolCall && assistantTextChunks.length > 0) {
|
if (!sawSpeakToolCall) {
|
||||||
const fallbackText = assistantTextChunks.join(" ").trim();
|
|
||||||
await this.ttsManager.generateAndWaitForPlayback(
|
|
||||||
fallbackText,
|
|
||||||
(msg) => this.emit(msg),
|
|
||||||
this.abortController.signal,
|
|
||||||
true
|
|
||||||
);
|
|
||||||
this.emit({
|
|
||||||
type: "activity_log",
|
|
||||||
payload: {
|
|
||||||
id: uuidv4(),
|
|
||||||
timestamp: new Date(),
|
|
||||||
type: "assistant",
|
|
||||||
content: fallbackText,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
this.sessionLogger.warn(
|
this.sessionLogger.warn(
|
||||||
{ voiceAssistantAgentId: agentId },
|
{ voiceAssistantAgentId: agentId },
|
||||||
"Voice agent responded without speak tool; used fallback TTS from assistant text"
|
"Voice agent turn completed without speak tool call; no audio playback emitted"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -389,7 +389,14 @@ export class VoiceAssistantWebSocketServer {
|
|||||||
? { sessionMessageType: message.message.type }
|
? { sessionMessageType: message.message.type }
|
||||||
: {}),
|
: {}),
|
||||||
};
|
};
|
||||||
this.logger.debug(messageSummary, "Received message");
|
const isSessionNoise =
|
||||||
|
message.type === "session" &&
|
||||||
|
(message.message.type === "client_heartbeat" ||
|
||||||
|
message.message.type === "voice_audio_chunk" ||
|
||||||
|
message.message.type === "dictation_stream_chunk");
|
||||||
|
if (!isSessionNoise) {
|
||||||
|
this.logger.debug(messageSummary, "Received message");
|
||||||
|
}
|
||||||
|
|
||||||
if (message.type === "ping") {
|
if (message.type === "ping") {
|
||||||
this.sendToClient(ws, { type: "pong" });
|
this.sendToClient(ws, { type: "pong" });
|
||||||
|
|||||||
@@ -348,6 +348,7 @@ export const SetVoiceModeMessageSchema = z.object({
|
|||||||
type: z.literal("set_voice_mode"),
|
type: z.literal("set_voice_mode"),
|
||||||
enabled: z.boolean(),
|
enabled: z.boolean(),
|
||||||
voiceAgentId: z.string().optional(),
|
voiceAgentId: z.string().optional(),
|
||||||
|
requestId: z.string().optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const SendAgentMessageSchema = z.object({
|
export const SendAgentMessageSchema = z.object({
|
||||||
@@ -571,6 +572,17 @@ export const SetAgentThinkingResponseMessageSchema = z.object({
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const SetVoiceModeResponseMessageSchema = z.object({
|
||||||
|
type: z.literal("set_voice_mode_response"),
|
||||||
|
payload: z.object({
|
||||||
|
requestId: z.string(),
|
||||||
|
enabled: z.boolean(),
|
||||||
|
voiceAgentId: z.string().nullable(),
|
||||||
|
accepted: z.boolean(),
|
||||||
|
error: z.string().nullable(),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
export const AgentPermissionResponseMessageSchema = z.object({
|
export const AgentPermissionResponseMessageSchema = z.object({
|
||||||
type: z.literal("agent_permission_response"),
|
type: z.literal("agent_permission_response"),
|
||||||
agentId: z.string(),
|
agentId: z.string(),
|
||||||
@@ -1627,6 +1639,7 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
|||||||
FetchAgentsResponseMessageSchema,
|
FetchAgentsResponseMessageSchema,
|
||||||
FetchAgentResponseMessageSchema,
|
FetchAgentResponseMessageSchema,
|
||||||
SendAgentMessageResponseMessageSchema,
|
SendAgentMessageResponseMessageSchema,
|
||||||
|
SetVoiceModeResponseMessageSchema,
|
||||||
SetAgentModeResponseMessageSchema,
|
SetAgentModeResponseMessageSchema,
|
||||||
SetAgentModelResponseMessageSchema,
|
SetAgentModelResponseMessageSchema,
|
||||||
SetAgentThinkingResponseMessageSchema,
|
SetAgentThinkingResponseMessageSchema,
|
||||||
@@ -1690,6 +1703,9 @@ export type FetchAgentResponseMessage = z.infer<
|
|||||||
export type SendAgentMessageResponseMessage = z.infer<
|
export type SendAgentMessageResponseMessage = z.infer<
|
||||||
typeof SendAgentMessageResponseMessageSchema
|
typeof SendAgentMessageResponseMessageSchema
|
||||||
>;
|
>;
|
||||||
|
export type SetVoiceModeResponseMessage = z.infer<
|
||||||
|
typeof SetVoiceModeResponseMessageSchema
|
||||||
|
>;
|
||||||
export type WaitForFinishResponseMessage = z.infer<
|
export type WaitForFinishResponseMessage = z.infer<
|
||||||
typeof WaitForFinishResponseMessageSchema
|
typeof WaitForFinishResponseMessageSchema
|
||||||
>;
|
>;
|
||||||
|
|||||||
Reference in New Issue
Block a user