jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add voice capability status reporting to client

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-14 13:14:57 +07:00
parent c4bebb97ab
commit a644adbf9b
16 changed files with 585 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/components/message-input.tsx
+35 -9
View File
@@ -32,7 +32,9 @@ import { DictationOverlay } from "./dictation-controls";
import { RealtimeVoiceOverlay } from "./realtime-voice-overlay";
import type { DaemonClient } from "@server/client/daemon-client";
import { usePanelStore } from "@/stores/panel-store";
import { useSessionStore } from "@/stores/session-store";
import { useVoiceOptional } from "@/contexts/voice-context";
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { Shortcut } from "@/components/ui/shortcut";
@@ -167,6 +169,17 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
const overlayTransition = useSharedValue(0);
const sendAfterTranscriptRef = useRef(false);
const valueRef = useRef(value);
const serverInfo = useSessionStore(
useCallback(
(state) => {
if (!voiceServerId) {
return null;
}
return state.sessions[voiceServerId]?.serverInfo ?? null;
},
[voiceServerId]
)
);
useEffect(() => {
valueRef.current = value;
@@ -212,10 +225,15 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
console.error("[MessageInput] Dictation error:", error);
}, []);
const dictationUnavailableMessage = resolveVoiceUnavailableMessage({
serverInfo,
mode: "dictation",
});
const canStartDictation = useCallback(() => {
const socketConnected = client?.isConnected ?? false;
return socketConnected && !disabled;
}, [client, disabled]);
return socketConnected && !disabled && !dictationUnavailableMessage;
}, [client, disabled, dictationUnavailableMessage]);
const canConfirmDictation = useCallback(() => {
const socketConnected = client?.isConnected ?? false;
@@ -268,6 +286,14 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
sendAfterTranscriptRef.current = false;
}, [dictationStatus, isDictating, isDictationProcessing]);
const startDictationIfAvailable = useCallback(async () => {
if (dictationUnavailableMessage) {
Alert.alert("Dictation unavailable", dictationUnavailableMessage);
return;
}
await startDictation();
}, [dictationUnavailableMessage, startDictation]);
// Cmd+D to start/submit dictation, Cmd+Shift+D toggles realtime voice, Escape cancels dictation
useEffect(() => {
if (!IS_WEB) return;
@@ -298,7 +324,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
}
void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
console.error("[MessageInput] Failed to start realtime voice", error);
const message = error instanceof Error ? error.message : "Voice features are not available right now.";
const message = error instanceof Error ? error.message : String(error);
Alert.alert("Voice unavailable", message);
});
};
@@ -348,7 +374,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
sendAfterTranscriptRef.current = true;
confirmDictation();
} else {
startDictation();
void startDictationIfAvailable();
}
return;
}
@@ -369,7 +395,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
isConnected,
isRealtimeVoiceForCurrentAgent,
isScreenFocused,
startDictation,
startDictationIfAvailable,
voiceAgentId,
voiceServerId,
voice,
@@ -400,13 +426,13 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
if (isDictating) {
await cancelDictation();
} else {
await startDictation();
await startDictationIfAvailable();
}
}, [
cancelDictation,
isDictating,
isRealtimeVoiceForCurrentAgent,
startDictation,
startDictationIfAvailable,
voice,
]);
@@ -464,7 +490,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
}
void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
console.error("[MessageInput] Failed to start realtime voice", error);
const message = error instanceof Error ? error.message : "Voice features are not available right now.";
const message = error instanceof Error ? error.message : String(error);
Alert.alert("Voice unavailable", message);
});
}, [
@@ -623,7 +649,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
sendAfterTranscriptRef.current = true;
confirmDictation();
} else {
startDictation();
void startDictationIfAvailable();
}
return;
}
packages/app/src/contexts/session-context.tsx
+8 -8
View File
@@ -18,6 +18,7 @@ import type {
AgentStreamEventPayload,
SessionOutboundMessage,
} from "@server/shared/messages";
import { parseServerInfoStatusPayload } from "@server/shared/messages";
import type { AgentLifecycleStatus } from "@server/shared/agent-lifecycle";
import type { AgentPermissionRequest } from "@server/server/agent/agent-sdk-types";
import type { DaemonClient, ConnectionState } from "@server/client/daemon-client";
@@ -984,15 +985,14 @@ export function SessionProvider({
const unsubStatus = client.on("status", (message) => {
if (message.type !== "status") return;
const status = message.payload.status;
if (status === "server_info") {
const payload = message.payload as any;
const rawServerId = typeof payload.serverId === "string" ? payload.serverId.trim() : "";
if (!rawServerId) return;
const rawHostname = typeof payload.hostname === "string" ? payload.hostname.trim() : "";
const serverInfo = parseServerInfoStatusPayload(message.payload);
if (serverInfo) {
updateSessionServerInfo(serverId, {
serverId: rawServerId,
hostname: rawHostname.length > 0 ? rawHostname : null,
serverId: serverInfo.serverId,
hostname: serverInfo.hostname,
...(serverInfo.capabilities
? { capabilities: serverInfo.capabilities }
: {}),
});
return;
}
packages/app/src/contexts/voice-context.tsx
+8
View File
@@ -2,6 +2,7 @@ import { createContext, useContext, useState, ReactNode, useCallback, useEffect,
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
import type { SessionState } from "@/stores/session-store";
import { useSessionStore } from "@/stores/session-store";
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
@@ -334,6 +335,13 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
if (!session) {
throw new Error(`Host ${serverId} is not connected`);
}
const unavailableMessage = resolveVoiceUnavailableMessage({
serverInfo: session.serverInfo,
mode: "voice",
});
if (unavailableMessage) {
throw new Error(unavailableMessage);
}
setIsVoiceSwitching(true);
voiceTransportReadyRef.current = false;
packages/app/src/dictation/dictation-stream-sender.ts
+5 -1
View File
@@ -86,7 +86,11 @@ export class DictationStreamSender {
}
if (!this.dictationId) {
void this.restartStream("enqueue");
if (!this.startPromise) {
void this.restartStream("enqueue").catch((error) => {
console.error("[DictationStreamSender] Failed to start stream from enqueue", error);
});
}
return;
}
packages/app/src/hooks/use-dictation.ts
+7 -3
View File
@@ -159,9 +159,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
if (!isRecordingRef.current) {
return;
}
void startNewStream("reconnect");
void startNewStream("reconnect").catch((error) => {
reportError(error, "Failed to restart dictation stream after reconnect");
});
});
}, [client, startNewStream]);
}, [client, reportError, startNewStream]);
useEffect(() => {
if (!client) {
@@ -260,7 +262,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
try {
await audio.start();
if (client?.isConnected) {
void startNewStream("start");
await startNewStream("start");
}
isRecordingRef.current = true;
setIsRecording(true);
@@ -268,9 +270,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
startDurationTracking();
}
} catch (err) {
await audio.stop().catch(() => undefined);
stopDurationTracking();
isRecordingRef.current = false;
setIsRecording(false);
setStatus("idle");
reportError(err, "Failed to start dictation");
} finally {
actionGateRef.current.starting = false;
packages/app/src/stores/session-store.ts
+21 -2
View File
@@ -21,6 +21,7 @@ import type {
FileDownloadTokenResponse,
GitSetupOptions,
ProjectPlacementPayload,
ServerCapabilities,
} from "@server/shared/messages";
import { isPerfLoggingEnabled, measurePayload, perfLog } from "@/utils/perf";
@@ -155,6 +156,7 @@ export interface DaemonConnectionSnapshot {
export type DaemonServerInfo = {
serverId: string;
hostname: string | null;
capabilities?: ServerCapabilities;
};
export interface AgentTimelineCursorState {
@@ -345,6 +347,13 @@ function createInitialSessionState(serverId: string, client: DaemonClient, audio
};
}
function areServerCapabilitiesEqual(
current: ServerCapabilities | undefined,
next: ServerCapabilities | undefined
): boolean {
return JSON.stringify(current ?? null) === JSON.stringify(next ?? null);
}
export const useSessionStore = create<SessionStore>()(
subscribeWithSelector((set, get) => ({
sessions: {},
@@ -446,8 +455,14 @@ export const useSessionStore = create<SessionStore>()(
const nextHostname = info.hostname?.trim() || null;
const prevHostname = session.serverInfo?.hostname?.trim() || null;
const nextCapabilities = info.capabilities;
const prevCapabilities = session.serverInfo?.capabilities;
if (session.serverInfo?.serverId === info.serverId && prevHostname === nextHostname) {
if (
session.serverInfo?.serverId === info.serverId &&
prevHostname === nextHostname &&
areServerCapabilitiesEqual(prevCapabilities, nextCapabilities)
) {
return prev;
}
@@ -462,7 +477,11 @@ export const useSessionStore = create<SessionStore>()(
...prev.sessions,
[serverId]: {
...session,
serverInfo: { serverId: info.serverId, hostname: nextHostname },
serverInfo: {
serverId: info.serverId,
hostname: nextHostname,
...(nextCapabilities ? { capabilities: nextCapabilities } : {}),
},
},
},
};
packages/app/src/utils/server-info-capabilities.test.ts
+118
View File
@@ -0,0 +1,118 @@
import { describe, expect, it } from "vitest";
import type { ServerCapabilities } from "@server/shared/messages";
import type { DaemonServerInfo } from "@/stores/session-store";
import {
getServerCapabilities,
getVoiceReadinessState,
resolveVoiceUnavailableMessage,
} from "./server-info-capabilities";
function buildServerInfo(capabilities?: ServerCapabilities): DaemonServerInfo {
return {
serverId: "srv-1",
hostname: "test-host",
...(capabilities ? { capabilities } : {}),
};
}
describe("server-info-capabilities", () => {
it("returns null capabilities when server_info does not include capability metadata", () => {
const serverInfo = buildServerInfo();
expect(getServerCapabilities({ serverInfo })).toBeNull();
});
it("returns the matching voice capability state by mode", () => {
const capabilities: ServerCapabilities = {
voice: {
dictation: {
enabled: true,
reason: "Dictation is warming up.",
},
voice: {
enabled: false,
reason: "Voice is disabled in daemon config.",
},
},
};
const serverInfo = buildServerInfo(capabilities);
expect(
getVoiceReadinessState({
serverInfo,
mode: "dictation",
})
).toEqual(capabilities.voice?.dictation);
expect(
getVoiceReadinessState({
serverInfo,
mode: "voice",
})
).toEqual(capabilities.voice?.voice);
});
it("returns null when capability is enabled and has no reason", () => {
const serverInfo = buildServerInfo({
voice: {
dictation: {
enabled: true,
reason: "",
},
voice: {
enabled: true,
reason: "",
},
},
});
expect(
resolveVoiceUnavailableMessage({
serverInfo,
mode: "dictation",
})
).toBeNull();
});
it("returns capability reason when present", () => {
const serverInfo = buildServerInfo({
voice: {
dictation: {
enabled: true,
reason: "Dictation models are still downloading.",
},
voice: {
enabled: true,
reason: "",
},
},
});
expect(
resolveVoiceUnavailableMessage({
serverInfo,
mode: "dictation",
})
).toBe("Dictation models are still downloading.");
});
it("returns null when capability reason is blank", () => {
const serverInfo = buildServerInfo({
voice: {
dictation: {
enabled: false,
reason: " ",
},
voice: {
enabled: true,
reason: "",
},
},
});
expect(
resolveVoiceUnavailableMessage({
serverInfo,
mode: "dictation",
})
).toBeNull();
});
});
packages/app/src/utils/server-info-capabilities.ts
+50
View File
@@ -0,0 +1,50 @@
import type { ServerCapabilityState } from "@server/shared/messages";
import type { DaemonServerInfo } from "@/stores/session-store";
export type VoiceReadinessMode = "dictation" | "voice";
export function getServerCapabilities(params: {
serverInfo: DaemonServerInfo | null | undefined;
}): DaemonServerInfo["capabilities"] | null {
const capabilities = params.serverInfo?.capabilities;
if (!capabilities) {
return null;
}
return capabilities;
}
export function getVoiceReadinessState(params: {
serverInfo: DaemonServerInfo | null | undefined;
mode: VoiceReadinessMode;
}): ServerCapabilityState | null {
const capabilities = getServerCapabilities({ serverInfo: params.serverInfo });
const voice = capabilities?.voice;
if (!voice) {
return null;
}
if (params.mode === "dictation") {
return voice.dictation;
}
return voice.voice;
}
export function resolveVoiceUnavailableMessage(params: {
serverInfo: DaemonServerInfo | null | undefined;
mode: VoiceReadinessMode;
}): string | null {
const readiness = getVoiceReadinessState({
serverInfo: params.serverInfo,
mode: params.mode,
});
if (!readiness) {
return null;
}
if (readiness.enabled && readiness.reason.trim().length === 0) {
return null;
}
const message = readiness.reason.trim();
if (message.length > 0) {
return message;
}
return null;
}
packages/app/src/utils/test-daemon-connection.ts
+5 -9
View File
@@ -1,5 +1,6 @@
import { DaemonClient } from "@server/client/daemon-client";
import type { DaemonClientConfig } from "@server/client/daemon-client";
import { parseServerInfoStatusPayload } from "@server/shared/messages";
import type { HostConnection } from "@/contexts/daemon-registry-context";
import { buildDaemonWebSocketUrl, buildRelayWebSocketUrl } from "./daemon-endpoints";
import { createTauriWebSocketTransportFactory } from "./tauri-daemon-transport";
@@ -116,15 +117,10 @@ function connectAndProbe(
unsubscribeStatus = client.on("status", (message) => {
if (message.type !== "status") return;
const payload = message.payload as { status?: unknown; serverId?: unknown; hostname?: unknown };
if (payload?.status !== "server_info") return;
const raw = typeof payload.serverId === "string" ? payload.serverId.trim() : "";
if (!raw) return;
serverId = raw;
hostname = typeof payload.hostname === "string" ? payload.hostname.trim() : null;
if (hostname && hostname.length === 0) {
hostname = null;
}
const payload = parseServerInfoStatusPayload(message.payload);
if (!payload) return;
serverId = payload.serverId;
hostname = payload.hostname;
maybeFinishOk();
});