Refactor voice mode to agent-only flow
This commit is contained in:
@@ -2,11 +2,8 @@ import { createContext, useContext, useState, ReactNode, useCallback, useEffect,
|
|||||||
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
|
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
|
||||||
import type { SessionState } from "@/stores/session-store";
|
import type { SessionState } from "@/stores/session-store";
|
||||||
import { useSessionStore } from "@/stores/session-store";
|
import { useSessionStore } from "@/stores/session-store";
|
||||||
import AsyncStorage from "@react-native-async-storage/async-storage";
|
|
||||||
import { randomUUID } from "expo-crypto";
|
|
||||||
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
|
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
|
||||||
|
|
||||||
const VOICE_CONVERSATION_ID_STORAGE_KEY = "@paseo:voice-conversation-id";
|
|
||||||
const KEEP_AWAKE_TAG = "paseo:voice";
|
const KEEP_AWAKE_TAG = "paseo:voice";
|
||||||
const VOICE_VAD_VOLUME_THRESHOLD = 0.18;
|
const VOICE_VAD_VOLUME_THRESHOLD = 0.18;
|
||||||
const VOICE_VAD_SILENCE_DURATION_MS = 1400;
|
const VOICE_VAD_SILENCE_DURATION_MS = 1400;
|
||||||
@@ -170,18 +167,9 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
console.log("[Voice] Mode enabled");
|
console.log("[Voice] Mode enabled");
|
||||||
|
|
||||||
if (session?.client) {
|
if (session?.client) {
|
||||||
let voiceConversationId =
|
await session.client.setVoiceMode(true);
|
||||||
(await AsyncStorage.getItem(VOICE_CONVERSATION_ID_STORAGE_KEY)) ?? null;
|
|
||||||
if (!voiceConversationId) {
|
|
||||||
voiceConversationId = randomUUID();
|
|
||||||
await AsyncStorage.setItem(
|
|
||||||
VOICE_CONVERSATION_ID_STORAGE_KEY,
|
|
||||||
voiceConversationId
|
|
||||||
);
|
|
||||||
}
|
|
||||||
await session.client.setVoiceConversation(true, voiceConversationId);
|
|
||||||
} else {
|
} else {
|
||||||
console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
|
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
console.error("[Voice] Failed to start:", error);
|
console.error("[Voice] Failed to start:", error);
|
||||||
@@ -204,9 +192,9 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
|||||||
console.log("[Voice] Mode disabled");
|
console.log("[Voice] Mode disabled");
|
||||||
|
|
||||||
if (session?.client) {
|
if (session?.client) {
|
||||||
await session.client.setVoiceConversation(false);
|
await session.client.setVoiceMode(false);
|
||||||
} else {
|
} else {
|
||||||
console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
|
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
console.error("[Voice] Failed to stop:", error);
|
console.error("[Voice] Failed to stop:", error);
|
||||||
|
|||||||
@@ -12,9 +12,7 @@ import type {
|
|||||||
AgentStreamEventPayload,
|
AgentStreamEventPayload,
|
||||||
AgentSnapshotPayload,
|
AgentSnapshotPayload,
|
||||||
AgentPermissionResolvedMessage,
|
AgentPermissionResolvedMessage,
|
||||||
VoiceConversationLoadedMessage,
|
|
||||||
CreateAgentRequestMessage,
|
CreateAgentRequestMessage,
|
||||||
DeleteVoiceConversationResponseMessage,
|
|
||||||
FileDownloadTokenResponse,
|
FileDownloadTokenResponse,
|
||||||
FileExplorerResponse,
|
FileExplorerResponse,
|
||||||
GitDiffResponse,
|
GitDiffResponse,
|
||||||
@@ -34,7 +32,6 @@ import type {
|
|||||||
ProjectIconResponse,
|
ProjectIconResponse,
|
||||||
ListCommandsResponse,
|
ListCommandsResponse,
|
||||||
ExecuteCommandResponse,
|
ExecuteCommandResponse,
|
||||||
ListVoiceConversationsResponseMessage,
|
|
||||||
ListProviderModelsResponseMessage,
|
ListProviderModelsResponseMessage,
|
||||||
SpeechModelsListResponse,
|
SpeechModelsListResponse,
|
||||||
SpeechModelsDownloadResponse,
|
SpeechModelsDownloadResponse,
|
||||||
@@ -184,9 +181,6 @@ export type CreateAgentRequestOptions = {
|
|||||||
labels?: Record<string, string>;
|
labels?: Record<string, string>;
|
||||||
} & AgentConfigOverrides;
|
} & AgentConfigOverrides;
|
||||||
|
|
||||||
type VoiceConversationLoadedPayload = VoiceConversationLoadedMessage["payload"];
|
|
||||||
type ListVoiceConversationsPayload = ListVoiceConversationsResponseMessage["payload"];
|
|
||||||
type DeleteVoiceConversationPayload = DeleteVoiceConversationResponseMessage["payload"];
|
|
||||||
type GitDiffPayload = GitDiffResponse["payload"];
|
type GitDiffPayload = GitDiffResponse["payload"];
|
||||||
type HighlightedDiffPayload = HighlightedDiffResponse["payload"];
|
type HighlightedDiffPayload = HighlightedDiffResponse["payload"];
|
||||||
type CheckoutStatusPayload = CheckoutStatusResponse["payload"];
|
type CheckoutStatusPayload = CheckoutStatusResponse["payload"];
|
||||||
@@ -926,87 +920,6 @@ export class DaemonClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Voice Conversation RPC
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
async loadVoiceConversation(
|
|
||||||
voiceConversationId: string,
|
|
||||||
requestId?: string
|
|
||||||
): Promise<VoiceConversationLoadedPayload> {
|
|
||||||
const resolvedRequestId = this.createRequestId(requestId);
|
|
||||||
const message = SessionInboundMessageSchema.parse({
|
|
||||||
type: "load_voice_conversation_request",
|
|
||||||
voiceConversationId,
|
|
||||||
requestId: resolvedRequestId,
|
|
||||||
});
|
|
||||||
return this.sendRequest({
|
|
||||||
requestId: resolvedRequestId,
|
|
||||||
message,
|
|
||||||
timeout: 10000,
|
|
||||||
options: { skipQueue: true },
|
|
||||||
select: (msg) => {
|
|
||||||
if (msg.type !== "voice_conversation_loaded") {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (msg.payload.requestId !== resolvedRequestId) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return msg.payload;
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async listVoiceConversations(requestId?: string): Promise<ListVoiceConversationsPayload> {
|
|
||||||
const resolvedRequestId = this.createRequestId(requestId);
|
|
||||||
const message = SessionInboundMessageSchema.parse({
|
|
||||||
type: "list_voice_conversations_request",
|
|
||||||
requestId: resolvedRequestId,
|
|
||||||
});
|
|
||||||
return this.sendRequest({
|
|
||||||
requestId: resolvedRequestId,
|
|
||||||
message,
|
|
||||||
timeout: 10000,
|
|
||||||
options: { skipQueue: true },
|
|
||||||
select: (msg) => {
|
|
||||||
if (msg.type !== "list_voice_conversations_response") {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (msg.payload.requestId !== resolvedRequestId) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return msg.payload;
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async deleteVoiceConversation(
|
|
||||||
voiceConversationId: string,
|
|
||||||
requestId?: string
|
|
||||||
): Promise<DeleteVoiceConversationPayload> {
|
|
||||||
const resolvedRequestId = this.createRequestId(requestId);
|
|
||||||
const message = SessionInboundMessageSchema.parse({
|
|
||||||
type: "delete_voice_conversation_request",
|
|
||||||
voiceConversationId,
|
|
||||||
requestId: resolvedRequestId,
|
|
||||||
});
|
|
||||||
return this.sendRequest({
|
|
||||||
requestId: resolvedRequestId,
|
|
||||||
message,
|
|
||||||
timeout: 10000,
|
|
||||||
options: { skipQueue: true },
|
|
||||||
select: (msg) => {
|
|
||||||
if (msg.type !== "delete_voice_conversation_response") {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (msg.payload.requestId !== resolvedRequestId) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return msg.payload;
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Agent Lifecycle
|
// Agent Lifecycle
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -1376,8 +1289,8 @@ export class DaemonClient {
|
|||||||
// Audio / Voice
|
// Audio / Voice
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
async setVoiceConversation(enabled: boolean, voiceConversationId?: string): Promise<void> {
|
async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise<void> {
|
||||||
this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId });
|
this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId });
|
||||||
}
|
}
|
||||||
|
|
||||||
async sendVoiceAudioChunk(
|
async sendVoiceAudioChunk(
|
||||||
|
|||||||
@@ -144,22 +144,18 @@ describe("daemon client E2E", () => {
|
|||||||
test("handles session actions", async () => {
|
test("handles session actions", async () => {
|
||||||
expect(ctx.client.isConnected).toBe(true);
|
expect(ctx.client.isConnected).toBe(true);
|
||||||
|
|
||||||
const voiceConversationId = randomUUID();
|
|
||||||
const loadResult = await ctx.client.loadVoiceConversation(voiceConversationId);
|
|
||||||
expect(loadResult.voiceConversationId).toBe(voiceConversationId);
|
|
||||||
expect(typeof loadResult.messageCount).toBe("number");
|
|
||||||
|
|
||||||
const agents = await ctx.client.fetchAgents();
|
const agents = await ctx.client.fetchAgents();
|
||||||
expect(Array.isArray(agents)).toBe(true);
|
expect(Array.isArray(agents)).toBe(true);
|
||||||
|
|
||||||
const listResult = await ctx.client.listVoiceConversations();
|
const voiceAgents = await ctx.client.fetchAgents({
|
||||||
expect(Array.isArray(listResult.conversations)).toBe(true);
|
filter: { labels: { surface: "voice" } },
|
||||||
|
});
|
||||||
|
expect(Array.isArray(voiceAgents)).toBe(true);
|
||||||
|
|
||||||
const missingId = randomUUID();
|
await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined();
|
||||||
const deleteResult = await ctx.client.deleteVoiceConversation(missingId);
|
await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined();
|
||||||
expect(deleteResult.voiceConversationId).toBe(missingId);
|
|
||||||
expect(deleteResult.success).toBe(false);
|
await ctx.client.deleteAgent(randomUUID());
|
||||||
expect(deleteResult.error).toBeTruthy();
|
|
||||||
}, 30000);
|
}, 30000);
|
||||||
|
|
||||||
test("emits server_info on websocket connect", async () => {
|
test("emits server_info on websocket connect", async () => {
|
||||||
@@ -185,19 +181,23 @@ describe("daemon client E2E", () => {
|
|||||||
await client.close();
|
await client.close();
|
||||||
}, 15000);
|
}, 15000);
|
||||||
|
|
||||||
test("matches request IDs for concurrent session requests", async () => {
|
test("handles concurrent filtered agent fetch requests", async () => {
|
||||||
const firstRequestId = `list-${Date.now()}-a`;
|
const firstRequestId = `fetch-${Date.now()}-a`;
|
||||||
const secondRequestId = `list-${Date.now()}-b`;
|
const secondRequestId = `fetch-${Date.now()}-b`;
|
||||||
|
|
||||||
const [first, second] = await Promise.all([
|
const [first, second] = await Promise.all([
|
||||||
ctx.client.listVoiceConversations(firstRequestId),
|
ctx.client.fetchAgents({
|
||||||
ctx.client.listVoiceConversations(secondRequestId),
|
requestId: firstRequestId,
|
||||||
|
filter: { labels: { surface: "voice" } },
|
||||||
|
}),
|
||||||
|
ctx.client.fetchAgents({
|
||||||
|
requestId: secondRequestId,
|
||||||
|
filter: { labels: { surface: "voice" } },
|
||||||
|
}),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
expect(Array.isArray(first.conversations)).toBe(true);
|
expect(Array.isArray(first)).toBe(true);
|
||||||
expect(Array.isArray(second.conversations)).toBe(true);
|
expect(Array.isArray(second)).toBe(true);
|
||||||
expect(first.requestId).toBe(firstRequestId);
|
|
||||||
expect(second.requestId).toBe(secondRequestId);
|
|
||||||
}, 15000);
|
}, 15000);
|
||||||
|
|
||||||
test(
|
test(
|
||||||
@@ -380,7 +380,7 @@ describe("daemon client E2E", () => {
|
|||||||
expect(sawAssistantMessage).toBe(true);
|
expect(sawAssistantMessage).toBe(true);
|
||||||
expect(sawRawAssistantMessage).toBe(true);
|
expect(sawRawAssistantMessage).toBe(true);
|
||||||
|
|
||||||
await ctx.client.setVoiceConversation(false);
|
await ctx.client.setVoiceMode(false);
|
||||||
|
|
||||||
await ctx.client.abortRequest();
|
await ctx.client.abortRequest();
|
||||||
await ctx.client.audioPlayed("audio-1");
|
await ctx.client.audioPlayed("audio-1");
|
||||||
@@ -594,7 +594,7 @@ describe("daemon client E2E", () => {
|
|||||||
test(
|
test(
|
||||||
"does not process non-voice turns through the voice agent path",
|
"does not process non-voice turns through the voice agent path",
|
||||||
async () => {
|
async () => {
|
||||||
await ctx.client.setVoiceConversation(false);
|
await ctx.client.setVoiceMode(false);
|
||||||
|
|
||||||
let sawTranscriptLog = false;
|
let sawTranscriptLog = false;
|
||||||
let sawAssistantChunk = false;
|
let sawAssistantChunk = false;
|
||||||
@@ -643,7 +643,7 @@ describe("daemon client E2E", () => {
|
|||||||
speechTest(
|
speechTest(
|
||||||
"voice mode buffers audio until isLast and emits transcription_result",
|
"voice mode buffers audio until isLast and emits transcription_result",
|
||||||
async () => {
|
async () => {
|
||||||
await ctx.client.setVoiceConversation(true, randomUUID());
|
await ctx.client.setVoiceMode(true, randomUUID());
|
||||||
|
|
||||||
const transcription = waitForSignal(30_000, (resolve) => {
|
const transcription = waitForSignal(30_000, (resolve) => {
|
||||||
const unsubscribe = ctx.client.on("transcription_result", (message) => {
|
const unsubscribe = ctx.client.on("transcription_result", (message) => {
|
||||||
@@ -738,7 +738,7 @@ describe("daemon client E2E", () => {
|
|||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
await Promise.allSettled([transcription, errorSignal]);
|
await Promise.allSettled([transcription, errorSignal]);
|
||||||
await ctx.client.setVoiceConversation(false);
|
await ctx.client.setVoiceMode(false);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
90_000
|
90_000
|
||||||
|
|||||||
@@ -182,7 +182,7 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(
|
|||||||
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
|
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
|
||||||
);
|
);
|
||||||
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
|
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
|
||||||
const VoiceConversationIdSchema = z.string().uuid();
|
const VoiceAgentIdSchema = z.string().uuid();
|
||||||
|
|
||||||
interface AudioBufferState {
|
interface AudioBufferState {
|
||||||
chunks: Buffer[];
|
chunks: Buffer[];
|
||||||
@@ -876,21 +876,6 @@ export class Session {
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case "load_voice_conversation_request":
|
|
||||||
await this.handleLoadVoiceConversation(msg.voiceConversationId, msg.requestId);
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "list_voice_conversations_request":
|
|
||||||
await this.handleListVoiceConversations(msg.requestId);
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "delete_voice_conversation_request":
|
|
||||||
await this.handleDeleteVoiceConversation(
|
|
||||||
msg.voiceConversationId,
|
|
||||||
msg.requestId
|
|
||||||
);
|
|
||||||
break;
|
|
||||||
|
|
||||||
case "delete_agent_request":
|
case "delete_agent_request":
|
||||||
await this.handleDeleteAgentRequest(msg.agentId, msg.requestId);
|
await this.handleDeleteAgentRequest(msg.agentId, msg.requestId);
|
||||||
break;
|
break;
|
||||||
@@ -899,8 +884,8 @@ export class Session {
|
|||||||
await this.handleArchiveAgentRequest(msg.agentId, msg.requestId);
|
await this.handleArchiveAgentRequest(msg.agentId, msg.requestId);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case "set_voice_conversation":
|
case "set_voice_mode":
|
||||||
await this.handleSetVoiceConversation(msg.enabled, msg.voiceConversationId);
|
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case "send_agent_message_request":
|
case "send_agent_message_request":
|
||||||
@@ -1155,138 +1140,6 @@ export class Session {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Load a voice conversation into this session (best-effort).
|
|
||||||
*/
|
|
||||||
public async handleLoadVoiceConversation(
|
|
||||||
voiceConversationId: string,
|
|
||||||
requestId: string
|
|
||||||
): Promise<void> {
|
|
||||||
const normalizedVoiceConversationId = this.parseVoiceConversationId(
|
|
||||||
voiceConversationId,
|
|
||||||
"load_voice_conversation_request"
|
|
||||||
);
|
|
||||||
this.voiceAssistantAgentId = normalizedVoiceConversationId;
|
|
||||||
|
|
||||||
this.emit({
|
|
||||||
type: "voice_conversation_loaded",
|
|
||||||
payload: {
|
|
||||||
voiceConversationId: normalizedVoiceConversationId,
|
|
||||||
messageCount: 0,
|
|
||||||
requestId,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* List all voice conversations
|
|
||||||
*/
|
|
||||||
public async handleListVoiceConversations(requestId: string): Promise<void> {
|
|
||||||
try {
|
|
||||||
const agents = await this.agentStorage.list();
|
|
||||||
const conversations = agents
|
|
||||||
.filter(
|
|
||||||
(agent) =>
|
|
||||||
agent.labels?.surface === "voice" &&
|
|
||||||
!agent.archivedAt &&
|
|
||||||
!agent.internal
|
|
||||||
)
|
|
||||||
.map((agent) => ({
|
|
||||||
id: agent.id,
|
|
||||||
lastUpdated: new Date(
|
|
||||||
agent.lastActivityAt ?? agent.updatedAt
|
|
||||||
).toISOString(),
|
|
||||||
messageCount: 0,
|
|
||||||
}))
|
|
||||||
.sort(
|
|
||||||
(a, b) =>
|
|
||||||
new Date(b.lastUpdated).getTime() -
|
|
||||||
new Date(a.lastUpdated).getTime()
|
|
||||||
);
|
|
||||||
this.emit({
|
|
||||||
type: "list_voice_conversations_response",
|
|
||||||
payload: {
|
|
||||||
conversations,
|
|
||||||
requestId,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (error: any) {
|
|
||||||
this.sessionLogger.error(
|
|
||||||
{ err: error },
|
|
||||||
"Failed to list voice conversations"
|
|
||||||
);
|
|
||||||
this.emit({
|
|
||||||
type: "activity_log",
|
|
||||||
payload: {
|
|
||||||
id: uuidv4(),
|
|
||||||
timestamp: new Date(),
|
|
||||||
type: "error",
|
|
||||||
content: `Failed to list voice conversations: ${error.message}`,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete a voice conversation
|
|
||||||
*/
|
|
||||||
public async handleDeleteVoiceConversation(
|
|
||||||
voiceConversationId: string,
|
|
||||||
requestId: string
|
|
||||||
): Promise<void> {
|
|
||||||
const normalizedVoiceConversationId = this.parseVoiceConversationId(
|
|
||||||
voiceConversationId,
|
|
||||||
"delete_voice_conversation_request"
|
|
||||||
);
|
|
||||||
try {
|
|
||||||
const record = await this.agentStorage.get(normalizedVoiceConversationId);
|
|
||||||
if (!record || record.labels?.surface !== "voice") {
|
|
||||||
this.emit({
|
|
||||||
type: "delete_voice_conversation_response",
|
|
||||||
payload: {
|
|
||||||
voiceConversationId: normalizedVoiceConversationId,
|
|
||||||
success: false,
|
|
||||||
error: "Voice conversation not found",
|
|
||||||
requestId,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const live = this.agentManager.getAgent(normalizedVoiceConversationId);
|
|
||||||
if (live) {
|
|
||||||
await this.agentManager.closeAgent(normalizedVoiceConversationId);
|
|
||||||
}
|
|
||||||
await this.agentStorage.remove(normalizedVoiceConversationId);
|
|
||||||
this.emit({
|
|
||||||
type: "delete_voice_conversation_response",
|
|
||||||
payload: {
|
|
||||||
voiceConversationId: normalizedVoiceConversationId,
|
|
||||||
success: true,
|
|
||||||
requestId,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
this.sessionLogger.info(
|
|
||||||
{ voiceConversationId: normalizedVoiceConversationId },
|
|
||||||
`Deleted voice conversation ${normalizedVoiceConversationId}`
|
|
||||||
);
|
|
||||||
} catch (error: any) {
|
|
||||||
this.sessionLogger.error(
|
|
||||||
{ err: error, voiceConversationId: normalizedVoiceConversationId },
|
|
||||||
`Failed to delete voice conversation ${normalizedVoiceConversationId}`
|
|
||||||
);
|
|
||||||
this.emit({
|
|
||||||
type: "delete_voice_conversation_response",
|
|
||||||
payload: {
|
|
||||||
voiceConversationId: normalizedVoiceConversationId,
|
|
||||||
success: false,
|
|
||||||
error: error.message,
|
|
||||||
requestId,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async handleRestartServerRequest(
|
private async handleRestartServerRequest(
|
||||||
requestId: string,
|
requestId: string,
|
||||||
reason?: string
|
reason?: string
|
||||||
@@ -1410,28 +1263,30 @@ export class Session {
|
|||||||
/**
|
/**
|
||||||
* Handle voice mode toggle
|
* Handle voice mode toggle
|
||||||
*/
|
*/
|
||||||
private async handleSetVoiceConversation(
|
private async handleSetVoiceMode(
|
||||||
enabled: boolean,
|
enabled: boolean,
|
||||||
voiceConversationId?: string
|
voiceAgentId?: string
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (enabled) {
|
if (enabled) {
|
||||||
const candidateVoiceConversationId =
|
let normalizedVoiceAgentId: string | null = null;
|
||||||
voiceConversationId && voiceConversationId.trim().length > 0
|
if (voiceAgentId && voiceAgentId.trim().length > 0) {
|
||||||
? voiceConversationId
|
normalizedVoiceAgentId = this.parseVoiceAgentId(
|
||||||
: uuidv4();
|
voiceAgentId,
|
||||||
const normalizedVoiceConversationId = this.parseVoiceConversationId(
|
"set_voice_mode"
|
||||||
candidateVoiceConversationId,
|
);
|
||||||
"set_voice_conversation"
|
} else {
|
||||||
);
|
normalizedVoiceAgentId = await this.findExistingVoiceAssistantAgentId();
|
||||||
|
}
|
||||||
|
|
||||||
this.isVoiceMode = true;
|
this.isVoiceMode = true;
|
||||||
this.voiceAssistantAgentId = normalizedVoiceConversationId;
|
this.voiceAssistantAgentId = normalizedVoiceAgentId;
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{
|
{
|
||||||
voiceAssistantAgentId: this.voiceAssistantAgentId,
|
voiceAssistantAgentId: this.voiceAssistantAgentId,
|
||||||
generatedConversationId: !voiceConversationId,
|
resumedVoiceAgent: Boolean(normalizedVoiceAgentId),
|
||||||
|
providedVoiceAgentId: Boolean(voiceAgentId?.trim()),
|
||||||
},
|
},
|
||||||
"Voice conversation enabled (agent-backed)"
|
"Voice mode enabled (agent-backed)"
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1439,18 +1294,45 @@ export class Session {
|
|||||||
this.isVoiceMode = false;
|
this.isVoiceMode = false;
|
||||||
this.sessionLogger.info(
|
this.sessionLogger.info(
|
||||||
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
|
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
|
||||||
"Voice conversation disabled (agent-backed)"
|
"Voice mode disabled (agent-backed)"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private parseVoiceConversationId(rawId: string, source: string): string {
|
private parseVoiceAgentId(rawId: string, source: string): string {
|
||||||
const parsed = VoiceConversationIdSchema.safeParse(rawId.trim());
|
const parsed = VoiceAgentIdSchema.safeParse(rawId.trim());
|
||||||
if (!parsed.success) {
|
if (!parsed.success) {
|
||||||
throw new Error(`${source}: voiceConversationId must be a UUID`);
|
throw new Error(`${source}: voiceAgentId must be a UUID`);
|
||||||
}
|
}
|
||||||
return parsed.data;
|
return parsed.data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async findExistingVoiceAssistantAgentId(): Promise<string | null> {
|
||||||
|
const scopedCandidates = await this.listAgentPayloads({
|
||||||
|
labels: {
|
||||||
|
surface: "voice",
|
||||||
|
voiceClientId: this.clientId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const fallbackCandidates = await this.listAgentPayloads({
|
||||||
|
labels: { surface: "voice" },
|
||||||
|
});
|
||||||
|
const available = [
|
||||||
|
...scopedCandidates,
|
||||||
|
...fallbackCandidates.filter(
|
||||||
|
(candidate) => !scopedCandidates.some((scoped) => scoped.id === candidate.id)
|
||||||
|
),
|
||||||
|
].filter((agent) => !agent.archivedAt);
|
||||||
|
if (available.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
available.sort((left, right) => {
|
||||||
|
const leftTime = Date.parse(left.updatedAt);
|
||||||
|
const rightTime = Date.parse(right.updatedAt);
|
||||||
|
return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0);
|
||||||
|
});
|
||||||
|
return available[0]?.id ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handle text message to agent (with optional image attachments)
|
* Handle text message to agent (with optional image attachments)
|
||||||
*/
|
*/
|
||||||
@@ -4562,7 +4444,7 @@ export class Session {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
|
// Set phase to LLM and process (TTS enabled in voice mode for voice agents)
|
||||||
this.clearSpeechInProgress("transcription complete");
|
this.clearSpeechInProgress("transcription complete");
|
||||||
this.setPhase("llm");
|
this.setPhase("llm");
|
||||||
this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, result.text);
|
this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, result.text);
|
||||||
@@ -4635,6 +4517,7 @@ export class Session {
|
|||||||
labels: {
|
labels: {
|
||||||
surface: "voice",
|
surface: "voice",
|
||||||
ui: "false",
|
ui: "false",
|
||||||
|
voiceClientId: this.clientId,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
this.voiceAssistantAgentId = created.id;
|
this.voiceAssistantAgentId = created.id;
|
||||||
|
|||||||
+2
-2
@@ -191,7 +191,7 @@ describe("speech models (download E2E)", () => {
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
await ctx.client.setVoiceConversation(true, randomUUID());
|
await ctx.client.setVoiceMode(true, randomUUID());
|
||||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||||
const isLast = offset + chunkBytes >= pcm16.length;
|
const isLast = offset + chunkBytes >= pcm16.length;
|
||||||
@@ -201,7 +201,7 @@ describe("speech models (download E2E)", () => {
|
|||||||
if (voiceText.length > 0) {
|
if (voiceText.length > 0) {
|
||||||
expect(voiceText).toContain("voice note");
|
expect(voiceText).toContain("voice note");
|
||||||
}
|
}
|
||||||
await ctx.client.setVoiceConversation(false);
|
await ctx.client.setVoiceMode(false);
|
||||||
|
|
||||||
// Streaming TTS: generate locally from downloaded model and validate chunking.
|
// Streaming TTS: generate locally from downloaded model and validate chunking.
|
||||||
const ttsText = "This is a voice note.";
|
const ttsText = "This is a voice note.";
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ function waitForSignal<T>(
|
|||||||
test(
|
test(
|
||||||
"routes voice turns through local agent speak tool",
|
"routes voice turns through local agent speak tool",
|
||||||
async () => {
|
async () => {
|
||||||
await ctx.client.setVoiceConversation(true, randomUUID());
|
await ctx.client.setVoiceMode(true, randomUUID());
|
||||||
|
|
||||||
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
|
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
|
||||||
const offAudio = ctx.client.on("audio_output", (msg) => {
|
const offAudio = ctx.client.on("audio_output", (msg) => {
|
||||||
|
|||||||
@@ -337,23 +337,6 @@ export const UnsubscribeAgentUpdatesMessageSchema = z.object({
|
|||||||
subscriptionId: z.string(),
|
subscriptionId: z.string(),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const LoadVoiceConversationRequestMessageSchema = z.object({
|
|
||||||
type: z.literal("load_voice_conversation_request"),
|
|
||||||
voiceConversationId: z.string(),
|
|
||||||
requestId: z.string(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const ListVoiceConversationsRequestMessageSchema = z.object({
|
|
||||||
type: z.literal("list_voice_conversations_request"),
|
|
||||||
requestId: z.string(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const DeleteVoiceConversationRequestMessageSchema = z.object({
|
|
||||||
type: z.literal("delete_voice_conversation_request"),
|
|
||||||
voiceConversationId: z.string(),
|
|
||||||
requestId: z.string(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const DeleteAgentRequestMessageSchema = z.object({
|
export const DeleteAgentRequestMessageSchema = z.object({
|
||||||
type: z.literal("delete_agent_request"),
|
type: z.literal("delete_agent_request"),
|
||||||
agentId: z.string(),
|
agentId: z.string(),
|
||||||
@@ -366,10 +349,10 @@ export const ArchiveAgentRequestMessageSchema = z.object({
|
|||||||
requestId: z.string(),
|
requestId: z.string(),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const SetVoiceConversationMessageSchema = z.object({
|
export const SetVoiceModeMessageSchema = z.object({
|
||||||
type: z.literal("set_voice_conversation"),
|
type: z.literal("set_voice_mode"),
|
||||||
enabled: z.boolean(),
|
enabled: z.boolean(),
|
||||||
voiceConversationId: z.string().optional(),
|
voiceAgentId: z.string().optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const SendAgentMessageSchema = z.object({
|
export const SendAgentMessageSchema = z.object({
|
||||||
@@ -884,12 +867,9 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
|
|||||||
FetchAgentRequestMessageSchema,
|
FetchAgentRequestMessageSchema,
|
||||||
SubscribeAgentUpdatesMessageSchema,
|
SubscribeAgentUpdatesMessageSchema,
|
||||||
UnsubscribeAgentUpdatesMessageSchema,
|
UnsubscribeAgentUpdatesMessageSchema,
|
||||||
LoadVoiceConversationRequestMessageSchema,
|
|
||||||
ListVoiceConversationsRequestMessageSchema,
|
|
||||||
DeleteVoiceConversationRequestMessageSchema,
|
|
||||||
DeleteAgentRequestMessageSchema,
|
DeleteAgentRequestMessageSchema,
|
||||||
ArchiveAgentRequestMessageSchema,
|
ArchiveAgentRequestMessageSchema,
|
||||||
SetVoiceConversationMessageSchema,
|
SetVoiceModeMessageSchema,
|
||||||
SendAgentMessageRequestSchema,
|
SendAgentMessageRequestSchema,
|
||||||
WaitForFinishRequestSchema,
|
WaitForFinishRequestSchema,
|
||||||
DictationStreamStartMessageSchema,
|
DictationStreamStartMessageSchema,
|
||||||
@@ -1127,15 +1107,6 @@ export const ArtifactMessageSchema = z.object({
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const VoiceConversationLoadedMessageSchema = z.object({
|
|
||||||
type: z.literal("voice_conversation_loaded"),
|
|
||||||
payload: z.object({
|
|
||||||
voiceConversationId: z.string(),
|
|
||||||
messageCount: z.number(),
|
|
||||||
requestId: z.string(),
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const AgentUpdateMessageSchema = z.object({
|
export const AgentUpdateMessageSchema = z.object({
|
||||||
type: z.literal("agent_update"),
|
type: z.literal("agent_update"),
|
||||||
payload: z.discriminatedUnion("kind", [
|
payload: z.discriminatedUnion("kind", [
|
||||||
@@ -1225,30 +1196,6 @@ export const WaitForFinishResponseMessageSchema = z.object({
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const ListVoiceConversationsResponseMessageSchema = z.object({
|
|
||||||
type: z.literal("list_voice_conversations_response"),
|
|
||||||
payload: z.object({
|
|
||||||
conversations: z.array(
|
|
||||||
z.object({
|
|
||||||
id: z.string(),
|
|
||||||
lastUpdated: z.string(),
|
|
||||||
messageCount: z.number(),
|
|
||||||
})
|
|
||||||
),
|
|
||||||
requestId: z.string(),
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const DeleteVoiceConversationResponseMessageSchema = z.object({
|
|
||||||
type: z.literal("delete_voice_conversation_response"),
|
|
||||||
payload: z.object({
|
|
||||||
voiceConversationId: z.string(),
|
|
||||||
success: z.boolean(),
|
|
||||||
error: z.string().optional(),
|
|
||||||
requestId: z.string(),
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const AgentPermissionRequestMessageSchema = z.object({
|
export const AgentPermissionRequestMessageSchema = z.object({
|
||||||
type: z.literal("agent_permission_request"),
|
type: z.literal("agent_permission_request"),
|
||||||
payload: z.object({
|
payload: z.object({
|
||||||
@@ -1679,7 +1626,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
|||||||
RpcErrorMessageSchema,
|
RpcErrorMessageSchema,
|
||||||
InitializeAgentResponseMessageSchema,
|
InitializeAgentResponseMessageSchema,
|
||||||
ArtifactMessageSchema,
|
ArtifactMessageSchema,
|
||||||
VoiceConversationLoadedMessageSchema,
|
|
||||||
AgentUpdateMessageSchema,
|
AgentUpdateMessageSchema,
|
||||||
AgentStreamMessageSchema,
|
AgentStreamMessageSchema,
|
||||||
AgentStreamSnapshotMessageSchema,
|
AgentStreamSnapshotMessageSchema,
|
||||||
@@ -1691,8 +1637,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
|||||||
SetAgentModelResponseMessageSchema,
|
SetAgentModelResponseMessageSchema,
|
||||||
SetAgentThinkingResponseMessageSchema,
|
SetAgentThinkingResponseMessageSchema,
|
||||||
WaitForFinishResponseMessageSchema,
|
WaitForFinishResponseMessageSchema,
|
||||||
ListVoiceConversationsResponseMessageSchema,
|
|
||||||
DeleteVoiceConversationResponseMessageSchema,
|
|
||||||
AgentPermissionRequestMessageSchema,
|
AgentPermissionRequestMessageSchema,
|
||||||
AgentPermissionResolvedMessageSchema,
|
AgentPermissionResolvedMessageSchema,
|
||||||
AgentDeletedMessageSchema,
|
AgentDeletedMessageSchema,
|
||||||
@@ -1737,9 +1681,6 @@ export type TranscriptionResultMessage = z.infer<typeof TranscriptionResultMessa
|
|||||||
export type StatusMessage = z.infer<typeof StatusMessageSchema>;
|
export type StatusMessage = z.infer<typeof StatusMessageSchema>;
|
||||||
export type RpcErrorMessage = z.infer<typeof RpcErrorMessageSchema>;
|
export type RpcErrorMessage = z.infer<typeof RpcErrorMessageSchema>;
|
||||||
export type ArtifactMessage = z.infer<typeof ArtifactMessageSchema>;
|
export type ArtifactMessage = z.infer<typeof ArtifactMessageSchema>;
|
||||||
export type VoiceConversationLoadedMessage = z.infer<
|
|
||||||
typeof VoiceConversationLoadedMessageSchema
|
|
||||||
>;
|
|
||||||
export type AgentUpdateMessage = z.infer<typeof AgentUpdateMessageSchema>;
|
export type AgentUpdateMessage = z.infer<typeof AgentUpdateMessageSchema>;
|
||||||
export type AgentStreamMessage = z.infer<typeof AgentStreamMessageSchema>;
|
export type AgentStreamMessage = z.infer<typeof AgentStreamMessageSchema>;
|
||||||
export type AgentStreamSnapshotMessage = z.infer<
|
export type AgentStreamSnapshotMessage = z.infer<
|
||||||
@@ -1758,12 +1699,6 @@ export type SendAgentMessageResponseMessage = z.infer<
|
|||||||
export type WaitForFinishResponseMessage = z.infer<
|
export type WaitForFinishResponseMessage = z.infer<
|
||||||
typeof WaitForFinishResponseMessageSchema
|
typeof WaitForFinishResponseMessageSchema
|
||||||
>;
|
>;
|
||||||
export type ListVoiceConversationsResponseMessage = z.infer<
|
|
||||||
typeof ListVoiceConversationsResponseMessageSchema
|
|
||||||
>;
|
|
||||||
export type DeleteVoiceConversationResponseMessage = z.infer<
|
|
||||||
typeof DeleteVoiceConversationResponseMessageSchema
|
|
||||||
>;
|
|
||||||
export type AgentPermissionRequestMessage = z.infer<typeof AgentPermissionRequestMessageSchema>;
|
export type AgentPermissionRequestMessage = z.infer<typeof AgentPermissionRequestMessageSchema>;
|
||||||
export type AgentPermissionResolvedMessage = z.infer<typeof AgentPermissionResolvedMessageSchema>;
|
export type AgentPermissionResolvedMessage = z.infer<typeof AgentPermissionResolvedMessageSchema>;
|
||||||
export type AgentDeletedMessage = z.infer<typeof AgentDeletedMessageSchema>;
|
export type AgentDeletedMessage = z.infer<typeof AgentDeletedMessageSchema>;
|
||||||
|
|||||||
Reference in New Issue
Block a user