jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add voice capability status reporting to client

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-14 13:47:07 +07:00
parent a644adbf9b
commit deeba6d6eb
5 changed files with 197 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/components/agent-input-area.tsx
+12 -4
View File
@@ -4,7 +4,6 @@ import {
Text,
ActivityIndicator,
Platform,
Alert,
} from "react-native";
import { useState, useEffect, useRef, useCallback, useMemo } from "react";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
@@ -32,6 +31,7 @@ import { encodeImages } from "@/utils/encode-images";
import { useKeyboardNavStore } from "@/stores/keyboard-nav-store";
import { focusWithRetries } from "@/utils/web-focus";
import { useVoiceOptional } from "@/contexts/voice-context";
import { useToast } from "@/contexts/toast-context";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { Shortcut } from "@/components/ui/shortcut";
@@ -80,6 +80,7 @@ export function AgentInputArea({
const client = useSessionStore(
(state) => state.sessions[serverId]?.client ?? null
);
const toast = useToast();
const voice = useVoiceOptional();
const isConnected = client?.isConnected ?? false;
@@ -470,10 +471,17 @@ export function AgentInputArea({
}
void voice.startVoice(serverId, agentId).catch((error) => {
console.error("[AgentInputArea] Failed to start voice mode", error);
const message = error instanceof Error ? error.message : "Voice features are not available right now.";
Alert.alert("Voice unavailable", message);
const message =
error instanceof Error
? error.message
: typeof error === "string"
? error
: null;
if (message && message.trim().length > 0) {
toast.error(message);
}
});
}, [agentId, isConnected, serverId, voice]);
}, [agentId, isConnected, serverId, toast, voice]);
function handleEditQueuedMessage(id: string) {
const item = queuedMessages.find((q) => q.id === id);
packages/app/src/components/message-input.tsx
+24 -7
View File
@@ -10,7 +10,6 @@ import {
Image,
Platform,
BackHandler,
Alert,
} from "react-native";
import {
useState,
@@ -34,6 +33,7 @@ import type { DaemonClient } from "@server/client/daemon-client";
import { usePanelStore } from "@/stores/panel-store";
import { useSessionStore } from "@/stores/session-store";
import { useVoiceOptional } from "@/contexts/voice-context";
import { useToast } from "@/contexts/toast-context";
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { Shortcut } from "@/components/ui/shortcut";
@@ -136,6 +136,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
ref
) {
const { theme } = useUnistyles();
const toast = useToast();
const voice = useVoiceOptional();
const toggleAgentList = usePanelStore((state) => state.toggleAgentList);
const toggleFileExplorer = usePanelStore((state) => state.toggleFileExplorer);
@@ -288,11 +289,11 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
const startDictationIfAvailable = useCallback(async () => {
if (dictationUnavailableMessage) {
Alert.alert("Dictation unavailable", dictationUnavailableMessage);
toast.error(dictationUnavailableMessage);
return;
}
await startDictation();
}, [dictationUnavailableMessage, startDictation]);
}, [dictationUnavailableMessage, startDictation, toast]);
// Cmd+D to start/submit dictation, Cmd+Shift+D toggles realtime voice, Escape cancels dictation
useEffect(() => {
@@ -324,8 +325,15 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
}
void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
console.error("[MessageInput] Failed to start realtime voice", error);
const message = error instanceof Error ? error.message : String(error);
Alert.alert("Voice unavailable", message);
const message =
error instanceof Error
? error.message
: typeof error === "string"
? error
: null;
if (message && message.trim().length > 0) {
toast.error(message);
}
});
};
@@ -396,6 +404,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
isRealtimeVoiceForCurrentAgent,
isScreenFocused,
startDictationIfAvailable,
toast,
voiceAgentId,
voiceServerId,
voice,
@@ -490,13 +499,21 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
}
void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
console.error("[MessageInput] Failed to start realtime voice", error);
const message = error instanceof Error ? error.message : String(error);
Alert.alert("Voice unavailable", message);
const message =
error instanceof Error
? error.message
: typeof error === "string"
? error
: null;
if (message && message.trim().length > 0) {
toast.error(message);
}
});
}, [
disabled,
handleStopRealtimeVoice,
isConnected,
toast,
voice,
voiceAgentId,
voiceServerId,
packages/server/src/server/daemon-client.e2e.test.ts
+51
View File
@@ -190,6 +190,57 @@ describe("daemon client E2E", () => {
await client.close();
}, 15000);
test("emits disabled voice capability reasons on fresh daemon startup", async () => {
const isolatedCtx = await createDaemonTestContext({
speech: {
providers: {
dictationStt: { provider: "local", explicit: true, enabled: false },
voiceStt: { provider: "local", explicit: true, enabled: false },
voiceTts: { provider: "local", explicit: true, enabled: false },
},
},
});
const client = new DaemonClient({
url: `ws://127.0.0.1:${isolatedCtx.daemon.port}/ws`,
});
try {
const infoPromise = waitForSignal<{
dictationEnabled: boolean;
dictationReason: string;
voiceEnabled: boolean;
voiceReason: string;
}>(5000, (resolve) => {
const unsubscribe = client.on("status", (message) => {
if (message.type !== "status") return;
const payload = parseServerInfoStatusPayload(message.payload);
if (!payload) return;
const voice = payload.capabilities?.voice;
if (!voice) return;
resolve({
dictationEnabled: voice.dictation.enabled,
dictationReason: voice.dictation.reason,
voiceEnabled: voice.voice.enabled,
voiceReason: voice.voice.reason,
});
});
return unsubscribe;
});
await client.connect();
const info = await infoPromise;
expect(info.dictationEnabled).toBe(false);
expect(info.dictationReason).toBe("Dictation is disabled in daemon config.");
expect(info.voiceEnabled).toBe(false);
expect(info.voiceReason).toBe("Realtime voice is disabled in daemon config.");
} finally {
await client.close().catch(() => undefined);
await isolatedCtx.cleanup();
}
}, 30000);
test("handles concurrent filtered agent fetch requests", async () => {
const firstRequestId = `fetch-${Date.now()}-a`;
const secondRequestId = `fetch-${Date.now()}-b`;
packages/server/src/server/websocket-server.relay-reconnect.test.ts
+70
View File
@@ -197,6 +197,47 @@ function createReadySpeechReadinessSnapshot(): SpeechReadinessSnapshot {
};
}
function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnapshot {
return {
generatedAt: "2026-02-14T00:00:00.000Z",
requiredLocalModelIds: [
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
],
missingLocalModelIds: [
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
],
download: {
inProgress: true,
error: null,
},
dictation: {
enabled: true,
available: false,
reasonCode: "stt_unavailable",
message: "Dictation is unavailable: speech-to-text service is not ready.",
retryable: false,
missingModelIds: [],
},
realtimeVoice: {
enabled: true,
available: false,
reasonCode: "stt_unavailable",
message: "Realtime voice is unavailable: speech-to-text service is not ready.",
retryable: false,
missingModelIds: [],
},
voiceFeature: {
enabled: true,
available: false,
reasonCode: "model_download_in_progress",
message:
"Voice features are unavailable while models download in the background (sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20).",
retryable: true,
missingModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
},
};
}
describe("relay external socket reconnect behavior", () => {
beforeEach(() => {
sessionMock.instances.length = 0;
@@ -318,6 +359,35 @@ describe("relay external socket reconnect behavior", () => {
await server.close();
});
test("includes temporary retry guidance while models are downloading", async () => {
const server = createServer();
const metadata: ExternalSocketMetadata = {
transport: "relay",
externalSessionKey: "relay:client-server-info-download-guidance",
};
const socket = new MockSocket();
await server.attachExternalSocket(socket, metadata);
expect(socket.sent).toHaveLength(1);
server.publishSpeechReadiness(createDownloadInProgressSpeechReadinessSnapshot());
expect(socket.sent).toHaveLength(2);
const envelope = JSON.parse(socket.sent[1] as string) as {
message?: { payload?: unknown };
};
const payload = parseServerInfoStatusPayload(envelope.message?.payload);
expect(payload?.capabilities?.voice?.dictation.enabled).toBe(true);
expect(payload?.capabilities?.voice?.voice.enabled).toBe(true);
expect(payload?.capabilities?.voice?.dictation.reason).toContain(
"Try again in a few minutes."
);
expect(payload?.capabilities?.voice?.voice.reason).toContain(
"Try again in a few minutes."
);
await server.close();
});
test("routes inbound binary mux frames to session.handleBinaryFrame", async () => {
const server = createServer();
const metadata: ExternalSocketMetadata = {
packages/server/src/server/websocket-server.ts
+40 -4
View File
@@ -55,14 +55,38 @@ type WebSocketServerConfig = {
};
function toServerCapabilityState(
state: SpeechReadinessSnapshot["dictation"]
params: {
state: SpeechReadinessSnapshot["dictation"];
reason: string;
}
): ServerCapabilityState {
const { state, reason } = params;
return {
enabled: state.enabled,
reason: state.available ? "" : state.message,
reason,
};
}
function resolveCapabilityReason(params: {
state: SpeechReadinessSnapshot["dictation"];
readiness: SpeechReadinessSnapshot;
}): string {
const { state, readiness } = params;
if (state.available) {
return "";
}
if (readiness.voiceFeature.reasonCode === "model_download_in_progress") {
const baseMessage = readiness.voiceFeature.message.trim();
if (baseMessage.includes("Try again in a few minutes")) {
return baseMessage;
}
return `${baseMessage} Try again in a few minutes.`;
}
return state.message;
}
function buildServerCapabilities(params: {
readiness: SpeechReadinessSnapshot | null;
}): ServerCapabilities | undefined {
@@ -72,8 +96,20 @@ function buildServerCapabilities(params: {
}
return {
voice: {
dictation: toServerCapabilityState(readiness.dictation),
voice: toServerCapabilityState(readiness.realtimeVoice),
dictation: toServerCapabilityState({
state: readiness.dictation,
reason: resolveCapabilityReason({
state: readiness.dictation,
readiness,
}),
}),
voice: toServerCapabilityState({
state: readiness.realtimeVoice,
reason: resolveCapabilityReason({
state: readiness.realtimeVoice,
readiness,
}),
}),
},
};
}