Merge pull request #29 from getpaseo/voice/background-downloading

Decouple onboarding from voice model downloads and gate unavailable voice starts

packages/app/src/components/agent-input-area.tsx

@@ -4,6 +4,7 @@ import {
   Text,
   ActivityIndicator,
   Platform,
+  Alert,
 } from "react-native";
 import { useState, useEffect, useRef, useCallback, useMemo } from "react";
 import { StyleSheet, useUnistyles } from "react-native-unistyles";
@@ -469,6 +470,8 @@ export function AgentInputArea({
     }
     void voice.startVoice(serverId, agentId).catch((error) => {
       console.error("[AgentInputArea] Failed to start voice mode", error);
+      const message = error instanceof Error ? error.message : "Voice features are not available right now.";
+      Alert.alert("Voice unavailable", message);
     });
   }, [agentId, isConnected, serverId, voice]);
 

packages/app/src/components/message-input.tsx

@@ -10,6 +10,7 @@ import {
   Image,
   Platform,
   BackHandler,
+  Alert,
 } from "react-native";
 import {
   useState,
@@ -297,6 +298,8 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
       }
       void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
         console.error("[MessageInput] Failed to start realtime voice", error);
+        const message = error instanceof Error ? error.message : "Voice features are not available right now.";
+        Alert.alert("Voice unavailable", message);
       });
     };
 
@@ -461,6 +464,8 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
     }
     void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
       console.error("[MessageInput] Failed to start realtime voice", error);
+      const message = error instanceof Error ? error.message : "Voice features are not available right now.";
+      Alert.alert("Voice unavailable", message);
     });
   }, [
     disabled,

packages/cli/src/commands/onboard.ts

@@ -3,11 +3,9 @@ import { Command } from 'commander'
 import { writeFileSync } from 'node:fs'
 import path from 'node:path'
 import {
-  ensureLocalSpeechModels,
   generateLocalPairingOffer,
   loadConfig,
   loadPersistedConfig,
-  type LocalSpeechModelId,
   type CliConfigOverrides,
   type PersistedConfig,
 } from '@getpaseo/server'
@@ -154,7 +152,7 @@ async function resolveVoiceSelection(mode: OnboardOptions['voice']): Promise<boo
   }
 
   const answer = await confirm({
-    message: 'Enable voice features? (downloads local STT/TTS models now)',
+    message: 'Enable voice features? (downloads local STT/TTS models in background)',
     active: 'Yes',
     inactive: 'No',
     initialValue: false,
@@ -172,221 +170,6 @@ type DownloadProgress = {
   pct: number | null
 }
 
-type LocalModelDownloadProgress = {
-  modelId: string | null
-  pct: number | null
-}
-
-type LocalSpeechDownloadLogger = {
-  child: (_bindings: Record<string, unknown>) => LocalSpeechDownloadLogger
-  info: (obj?: unknown, msg?: string) => void
-  error: (_obj?: unknown, _msg?: string) => void
-}
-
-type LocalSpeechDownloadEvent =
-  | {
-      type: 'progress'
-      progress: LocalModelDownloadProgress
-    }
-  | {
-      type: 'phase'
-      phase: 'extracting' | 'verifying' | 'finalizing' | 'completed'
-    }
-
-function resolveRequiredLocalModelIds(config: ReturnType<typeof loadConfig>): LocalSpeechModelId[] {
-  const providers = config.speech?.providers
-  const local = config.speech?.local
-
-  if (!providers || !local) {
-    return []
-  }
-
-  const ids = new Set<LocalSpeechModelId>()
-
-  if (providers.dictationStt.enabled !== false && providers.dictationStt.provider === 'local') {
-    ids.add(local.models.dictationStt)
-  }
-  if (providers.voiceStt.enabled !== false && providers.voiceStt.provider === 'local') {
-    ids.add(local.models.voiceStt)
-  }
-  if (providers.voiceTts.enabled !== false && providers.voiceTts.provider === 'local') {
-    ids.add(local.models.voiceTts)
-  }
-
-  return Array.from(ids)
-}
-
-function parseLocalModelDownloadProgress(payload: unknown): LocalModelDownloadProgress | null {
-  if (!payload || typeof payload !== 'object') {
-    return null
-  }
-
-  const value = payload as Record<string, unknown>
-  const modelId = typeof value.modelId === 'string' ? value.modelId : null
-  const pctRaw = value.pct
-  const pct = typeof pctRaw === 'number' && Number.isFinite(pctRaw) ? Math.max(0, Math.min(100, Math.floor(pctRaw))) : null
-
-  return {
-    modelId,
-    pct,
-  }
-}
-
-function renderLocalModelProgress(params: {
-  modelId: LocalSpeechModelId
-  modelIndex: number
-  modelCount: number
-  pct: number | null
-}): string {
-  const prefix = `Downloading speech model ${params.modelIndex}/${params.modelCount}: ${params.modelId}`
-  if (params.pct === null) {
-    return `${prefix}...`
-  }
-  return `${prefix} (${params.pct}%)`
-}
-
-function createLocalSpeechDownloadLogger(
-  onEvent: (event: LocalSpeechDownloadEvent) => void
-): LocalSpeechDownloadLogger {
-  const logger: LocalSpeechDownloadLogger = {
-    child: () => logger,
-    info: (obj?: unknown, msg?: string) => {
-      if (msg === 'Downloading model artifact') {
-        const progress = parseLocalModelDownloadProgress(obj)
-        if (!progress) {
-          return
-        }
-        onEvent({
-          type: 'progress',
-          progress,
-        })
-        return
-      }
-      if (msg === 'Extracting model archive') {
-        onEvent({ type: 'phase', phase: 'extracting' })
-        return
-      }
-      if (msg === 'Verifying downloaded model files') {
-        onEvent({ type: 'phase', phase: 'verifying' })
-        return
-      }
-      if (msg === 'Finalizing model artifacts') {
-        onEvent({ type: 'phase', phase: 'finalizing' })
-        return
-      }
-      if (msg === 'Model download completed') {
-        onEvent({ type: 'phase', phase: 'completed' })
-        return
-      }
-    },
-    error: () => {
-      // no-op: onboarding handles surfaced errors from ensureLocalSpeechModels.
-    },
-  }
-  return logger
-}
-
-async function prepareLocalSpeechModelsBeforeStart(args: {
-  config: ReturnType<typeof loadConfig>
-  richUi: boolean
-}): Promise<void> {
-  const local = args.config.speech?.local
-  const modelIds = resolveRequiredLocalModelIds(args.config)
-
-  if (!local || modelIds.length === 0) {
-    return
-  }
-
-  if (local.autoDownload === false) {
-    log.warn('Local speech model auto-download is disabled. Voice may be unavailable until models are installed.')
-    return
-  }
-
-  const modelList = modelIds.join(', ')
-  const modelCount = modelIds.length
-  const downloadSpinner = args.richUi ? spinner() : null
-  let lastPlainStatus = ''
-
-  const emitStatus = (status: string): void => {
-    if (downloadSpinner) {
-      downloadSpinner.message(status)
-      return
-    }
-    if (status === lastPlainStatus) {
-      return
-    }
-    console.log(status)
-    lastPlainStatus = status
-  }
-
-  if (downloadSpinner) {
-    downloadSpinner.start(`Preparing local speech models (${modelCount})...`)
-  } else {
-    log.message(`Preparing local speech models (${modelCount}): ${modelList}`)
-  }
-
-  try {
-    for (const [index, modelId] of modelIds.entries()) {
-      const modelIndex = index + 1
-      emitStatus(`Checking speech model ${modelIndex}/${modelCount}: ${modelId}`)
-
-      const perModelLogger = createLocalSpeechDownloadLogger((event) => {
-        if (event.type === 'progress') {
-          const progress = event.progress
-          if (progress.modelId && progress.modelId !== modelId) {
-            return
-          }
-          emitStatus(
-            renderLocalModelProgress({
-              modelId,
-              modelIndex,
-              modelCount,
-              pct: progress.pct,
-            })
-          )
-          return
-        }
-
-        if (event.phase === 'extracting') {
-          emitStatus(`Extracting speech model ${modelIndex}/${modelCount}: ${modelId}`)
-          return
-        }
-        if (event.phase === 'verifying') {
-          emitStatus(`Verifying speech model ${modelIndex}/${modelCount}: ${modelId}`)
-          return
-        }
-        if (event.phase === 'finalizing') {
-          emitStatus(`Finalizing speech model ${modelIndex}/${modelCount}: ${modelId}`)
-          return
-        }
-      })
-
-      await ensureLocalSpeechModels({
-        modelsDir: local.modelsDir,
-        modelIds: [modelId],
-        autoDownload: true,
-        logger: perModelLogger as any,
-      })
-
-      emitStatus(`Speech model ready ${modelIndex}/${modelCount}: ${modelId}`)
-    }
-
-    if (downloadSpinner) {
-      downloadSpinner.stop(`Local speech models ready (${modelCount})`)
-    } else {
-      log.message(`Local speech models ready (${modelCount}): ${modelList}`)
-    }
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error)
-    if (downloadSpinner) {
-      downloadSpinner.error(`Failed to prepare local speech models: ${message}`)
-    } else {
-      log.error(`Failed to prepare local speech models: ${message}`)
-    }
-    throw error
-  }
-}
-
 function parseDownloadProgress(logTail: string): DownloadProgress | null {
   const lines = logTail.split('\n').filter(Boolean)
 
@@ -583,19 +366,10 @@ export async function runOnboard(options: OnboardOptions): Promise<void> {
   const config = loadConfig(paseoHome, { cli: toCliOverrides(options) })
 
   const voiceStatus = voiceEnabled
-    ? 'Voice features enabled. Local speech models will be downloaded if missing.'
+    ? 'Voice features enabled. Local speech models will download in the background if missing.'
-    : 'Voice features disabled. Local speech models will not be downloaded now.'
+    : 'Voice features disabled. Local speech models will not be downloaded.'
   log.message(voiceStatus)
 
-  try {
-    await prepareLocalSpeechModelsBeforeStart({
-      config,
-      richUi,
-    })
-  } catch {
-    process.exit(1)
-  }
-
   const stateBeforeStart = resolveLocalDaemonState({ home: options.home })
   const startSpinner = richUi ? spinner() : null
 

packages/server/src/client/daemon-client.ts

@@ -1485,7 +1485,7 @@ export class DaemonClient {
       ...(agentId ? { agentId } : {}),
       requestId,
     });
-    return this.sendRequest({
+    const response = await this.sendRequest({
       requestId,
       message,
       timeout: 10000,
@@ -1499,6 +1499,14 @@ export class DaemonClient {
         return msg.payload;
       },
     });
+    if (!response.accepted) {
+      const codeSuffix =
+        typeof response.reasonCode === "string" && response.reasonCode.trim().length > 0
+          ? ` (${response.reasonCode})`
+          : "";
+      throw new Error((response.error ?? "Failed to set voice mode") + codeSuffix);
+    }
+    return response;
   }
 
   async sendVoiceAudioChunk(

packages/server/src/server/agent/stt-manager.ts

@@ -1,5 +1,6 @@
 import type pino from "pino";
 import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js";
+import { toResolver, type Resolvable } from "../speech/provider-resolver.js";
 import { maybePersistDebugAudio } from "./stt-debug.js";
 import { parsePcm16MonoWav, parsePcmRateFromFormat } from "../speech/audio.js";
 import { Pcm16MonoResampler } from "./pcm16-resampler.js";
@@ -39,12 +40,16 @@ export interface SessionTranscriptionResult extends TranscriptionResult {
 export class STTManager {
   private readonly sessionId: string;
   private readonly logger: pino.Logger;
-  private readonly stt: SpeechToTextProvider | null;
+  private readonly resolveStt: () => SpeechToTextProvider | null;
 
-  constructor(sessionId: string, logger: pino.Logger, stt: SpeechToTextProvider | null) {
+  constructor(
+    sessionId: string,
+    logger: pino.Logger,
+    stt: Resolvable<SpeechToTextProvider | null>
+  ) {
     this.sessionId = sessionId;
     this.logger = logger.child({ module: "agent", component: "stt-manager", sessionId });
-    this.stt = stt;
+    this.resolveStt = toResolver(stt);
   }
 
   /**
@@ -55,7 +60,8 @@ export class STTManager {
     format: string,
     metadata?: TranscriptionMetadata
   ): Promise<SessionTranscriptionResult> {
-    if (!this.stt) {
+    const stt = this.resolveStt();
+    if (!stt) {
       throw new Error("STT not configured");
     }
 
@@ -81,7 +87,7 @@ export class STTManager {
       this.logger.warn({ err: error }, "Failed to persist debug audio");
     }
 
-    const session = this.stt.createSession({
+    const session = stt.createSession({
       logger: this.logger.child({ component: "stt-session" }),
       language: "en",
     });

packages/server/src/server/agent/tts-manager.ts

@@ -1,6 +1,7 @@
 import type pino from "pino";
 import { v4 as uuidv4 } from "uuid";
 import type { TextToSpeechProvider } from "../speech/speech-provider.js";
+import { toResolver, type Resolvable } from "../speech/provider-resolver.js";
 import type { SessionOutboundMessage } from "../messages.js";
 
 interface PendingPlayback {
@@ -95,11 +96,15 @@ function splitTextForTts(text: string, maxChars: number): string[] {
 export class TTSManager {
   private pendingPlaybacks: Map<string, PendingPlayback> = new Map();
   private readonly logger: pino.Logger;
-  private readonly tts: TextToSpeechProvider | null;
+  private readonly resolveTts: () => TextToSpeechProvider | null;
 
-  constructor(sessionId: string, logger: pino.Logger, tts: TextToSpeechProvider | null) {
+  constructor(
+    sessionId: string,
+    logger: pino.Logger,
+    tts: Resolvable<TextToSpeechProvider | null>
+  ) {
     this.logger = logger.child({ module: "agent", component: "tts-manager", sessionId });
-    this.tts = tts;
+    this.resolveTts = toResolver(tts);
   }
 
   /**
@@ -143,7 +148,8 @@ export class TTSManager {
     abortSignal: AbortSignal,
     isVoiceMode: boolean
   ): Promise<void> {
-    if (!this.tts) {
+    const tts = this.resolveTts();
+    if (!tts) {
       throw new Error("TTS not configured");
     }
 
@@ -153,7 +159,7 @@ export class TTSManager {
     }
 
     // Generate TTS audio stream
-    const { stream, format } = await this.tts.synthesizeSpeech(text);
+    const { stream, format } = await tts.synthesizeSpeech(text);
 
     if (abortSignal.aborted) {
       this.logger.debug("Aborted after generating audio");

packages/server/src/server/bootstrap.ts

@@ -457,9 +457,10 @@ export async function createPaseoDaemon(
     },
   });
   const {
-    sttService,
+    resolveVoiceStt,
-    ttsService,
+    resolveVoiceTts,
-    dictationSttService,
+    resolveDictationStt,
+    getSpeechReadiness,
     cleanup: cleanupSpeechRuntime,
     localModelConfig,
   } = await initializeSpeechRuntime({
@@ -478,7 +479,7 @@ export async function createPaseoDaemon(
     config.paseoHome,
     createInMemoryAgentMcpTransport,
     { allowedOrigins, allowedHosts: config.allowedHosts },
-    { stt: sttService, tts: ttsService },
+    { stt: resolveVoiceStt, tts: resolveVoiceTts },
     terminalManager,
     {
       voiceAgentMcpStdio: {
@@ -496,8 +497,9 @@ export async function createPaseoDaemon(
     },
     {
       finalTimeoutMs: config.dictationFinalTimeoutMs,
-      stt: dictationSttService,
+      stt: resolveDictationStt,
       localModels: localModelConfig ?? undefined,
+      getSpeechReadiness,
     },
     config.agentProviderSettings
   );

packages/server/src/server/dictation/dictation-stream-manager.ts

@@ -11,6 +11,7 @@ import type {
   SpeechToTextProvider,
   StreamingTranscriptionSession,
 } from "../speech/speech-provider.js";
+import { toResolver, type Resolvable } from "../speech/provider-resolver.js";
 import { parsePcmRateFromFormat, pcm16lePeakAbs } from "../speech/audio.js";
 
 const PCM_CHANNELS = 1;
@@ -117,7 +118,7 @@ export class DictationStreamManager {
   private readonly logger: pino.Logger;
   private readonly emit: (msg: DictationStreamOutboundMessage) => void;
   private readonly sessionId: string;
-  private readonly stt: SpeechToTextProvider | null;
+  private readonly resolveStt: () => SpeechToTextProvider | null;
   private readonly finalTimeoutMs: number;
   private readonly autoCommitSeconds: number;
   private readonly streams = new Map<string, DictationStreamState>();
@@ -126,14 +127,14 @@ export class DictationStreamManager {
     logger: pino.Logger;
     emit: (msg: DictationStreamOutboundMessage) => void;
     sessionId: string;
-    stt: SpeechToTextProvider | null;
+    stt: Resolvable<SpeechToTextProvider | null>;
     finalTimeoutMs?: number;
     autoCommitSeconds?: number;
   }) {
     this.logger = params.logger.child({ component: "dictation-stream-manager" });
     this.emit = params.emit;
     this.sessionId = params.sessionId;
-    this.stt = params.stt;
+    this.resolveStt = toResolver(params.stt);
     this.finalTimeoutMs = params.finalTimeoutMs ?? DEFAULT_DICTATION_FINAL_TIMEOUT_MS;
     this.autoCommitSeconds =
       params.autoCommitSeconds ??
@@ -150,7 +151,8 @@ export class DictationStreamManager {
   public async handleStart(dictationId: string, format: string): Promise<void> {
     this.cleanupDictationStream(dictationId);
 
-    if (!this.stt) {
+    const sttProvider = this.resolveStt();
+    if (!sttProvider) {
       this.failDictationStream(dictationId, "Dictation STT not configured", false);
       return;
     }
@@ -159,11 +161,18 @@ export class DictationStreamManager {
       process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT ??
       "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript.";
 
-    const stt = this.stt.createSession({
+    let stt: ReturnType<SpeechToTextProvider["createSession"]>;
-      logger: this.logger.child({ dictationId }),
+    try {
-      language: "en",
+      stt = sttProvider.createSession({
-      prompt: transcriptionPrompt,
+        logger: this.logger.child({ dictationId }),
-    });
+        language: "en",
+        prompt: transcriptionPrompt,
+      });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.failDictationStream(dictationId, message, false);
+      return;
+    }
 
     stt.on("committed", ({ segmentId }) => {
       const state = this.streams.get(dictationId);
@@ -221,7 +230,18 @@ export class DictationStreamManager {
       void this.failAndCleanupDictationStream(dictationId, message, true);
     });
 
-    await stt.connect();
+    try {
+      await stt.connect();
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.failDictationStream(dictationId, message, true);
+      try {
+        stt.close();
+      } catch {
+        // no-op
+      }
+      return;
+    }
 
     const inputRate = parsePcmRateFromFormat(format, 16000) ?? 16000;
     if (!Number.isFinite(inputRate) || inputRate <= 0) {

packages/server/src/server/session.ts

@@ -135,6 +135,8 @@ import {
   listLocalSpeechModels,
   type LocalSpeechModelId,
 } from "./speech/providers/local/models.js";
+import type { Resolvable } from "./speech/provider-resolver.js";
+import type { SpeechReadinessSnapshot, SpeechReadinessState } from "./speech/speech-runtime.js";
 import type pino from "pino";
 
 const execAsync = promisify(exec);
@@ -328,8 +330,8 @@ export type SessionOptions = {
   agentManager: AgentManager;
   agentStorage: AgentStorage;
   createAgentMcpTransport: AgentMcpTransportFactory;
-  stt: SpeechToTextProvider | null;
+  stt: Resolvable<SpeechToTextProvider | null>;
-  tts: TextToSpeechProvider | null;
+  tts: Resolvable<TextToSpeechProvider | null>;
   terminalManager: TerminalManager | null;
   voice?: {
     voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
@@ -344,15 +346,43 @@ export type SessionOptions = {
   };
   dictation?: {
     finalTimeoutMs?: number;
-    stt?: SpeechToTextProvider | null;
+    stt?: Resolvable<SpeechToTextProvider | null>;
     localModels?: {
       modelsDir: string;
       defaultModelIds: LocalSpeechModelId[];
     };
+    getSpeechReadiness?: () => SpeechReadinessSnapshot;
   };
   agentProviderRuntimeSettings?: AgentProviderRuntimeSettingsMap;
 };
 
+type VoiceFeatureUnavailableContext = {
+  reasonCode: SpeechReadinessSnapshot["voiceFeature"]["reasonCode"];
+  message: string;
+  retryable: boolean;
+  missingModelIds: LocalSpeechModelId[];
+};
+
+type VoiceFeatureUnavailableResponseMetadata = {
+  reasonCode?: SpeechReadinessSnapshot["voiceFeature"]["reasonCode"];
+  retryable?: boolean;
+  missingModelIds?: LocalSpeechModelId[];
+};
+
+class VoiceFeatureUnavailableError extends Error {
+  readonly reasonCode: SpeechReadinessSnapshot["voiceFeature"]["reasonCode"];
+  readonly retryable: boolean;
+  readonly missingModelIds: LocalSpeechModelId[];
+
+  constructor(context: VoiceFeatureUnavailableContext) {
+    super(context.message);
+    this.name = "VoiceFeatureUnavailableError";
+    this.reasonCode = context.reasonCode;
+    this.retryable = context.retryable;
+    this.missingModelIds = [...context.missingModelIds];
+  }
+}
+
 function convertPCMToWavBuffer(
   pcmBuffer: Buffer,
   sampleRate: number,
@@ -538,6 +568,7 @@ export class Session {
   private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
   private readonly ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
   private readonly removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
+  private readonly getSpeechReadiness?: () => SpeechReadinessSnapshot;
   private readonly agentProviderRuntimeSettings: AgentProviderRuntimeSettingsMap | undefined;
   private voiceModeAgentId: string | null = null;
   private voiceModeBaseConfig: VoiceModeBaseConfig | null = null;
@@ -589,6 +620,7 @@ export class Session {
     this.unregisterVoiceCallerContext = voiceBridge?.unregisterVoiceCallerContext;
     this.ensureVoiceMcpSocketForAgent = voiceBridge?.ensureVoiceMcpSocketForAgent;
     this.removeVoiceMcpSocketForAgent = voiceBridge?.removeVoiceMcpSocketForAgent;
+    this.getSpeechReadiness = dictation?.getSpeechReadiness;
     this.agentProviderRuntimeSettings = agentProviderRuntimeSettings;
     this.abortController = new AbortController();
     this.sessionLogger = logger.child({
@@ -1197,6 +1229,22 @@ export class Session {
           break;
 
         case "dictation_stream_start":
+          {
+            const unavailable = this.resolveVoiceFeatureUnavailableContext("dictation");
+            if (unavailable) {
+              this.emit({
+                type: "dictation_stream_error",
+                payload: {
+                  dictationId: msg.dictationId,
+                  error: unavailable.message,
+                  retryable: unavailable.retryable,
+                  reasonCode: unavailable.reasonCode,
+                  missingModelIds: unavailable.missingModelIds,
+                },
+              });
+              break;
+            }
+          }
           await this.dictationStreamManager.handleStart(msg.dictationId, msg.format);
           break;
 
@@ -1721,6 +1769,61 @@ export class Session {
     }
   }
 
+  private toVoiceFeatureUnavailableContext(
+    state: SpeechReadinessState
+  ): VoiceFeatureUnavailableContext {
+    return {
+      reasonCode: state.reasonCode,
+      message: state.message,
+      retryable: state.retryable,
+      missingModelIds: [...state.missingModelIds],
+    };
+  }
+
+  private resolveModeReadinessState(
+    readiness: SpeechReadinessSnapshot,
+    mode: "voice_mode" | "dictation"
+  ): SpeechReadinessState {
+    if (mode === "voice_mode") {
+      return readiness.realtimeVoice;
+    }
+    return readiness.dictation;
+  }
+
+  private getVoiceFeatureUnavailableResponseMetadata(
+    error: unknown
+  ): VoiceFeatureUnavailableResponseMetadata {
+    if (!(error instanceof VoiceFeatureUnavailableError)) {
+      return {};
+    }
+    return {
+      reasonCode: error.reasonCode,
+      retryable: error.retryable,
+      missingModelIds: error.missingModelIds,
+    };
+  }
+
+  private resolveVoiceFeatureUnavailableContext(
+    mode: "voice_mode" | "dictation"
+  ): VoiceFeatureUnavailableContext | null {
+    const readiness = this.getSpeechReadiness?.();
+    if (!readiness) {
+      return null;
+    }
+
+    const modeReadiness = this.resolveModeReadinessState(readiness, mode);
+    if (!modeReadiness.enabled) {
+      return this.toVoiceFeatureUnavailableContext(modeReadiness);
+    }
+    if (!readiness.voiceFeature.available) {
+      return this.toVoiceFeatureUnavailableContext(readiness.voiceFeature);
+    }
+    if (!modeReadiness.available) {
+      return this.toVoiceFeatureUnavailableContext(modeReadiness);
+    }
+    return null;
+  }
+
   /**
    * Handle voice mode toggle
    */
@@ -1731,6 +1834,11 @@ export class Session {
   ): Promise<void> {
     try {
       if (enabled) {
+        const unavailable = this.resolveVoiceFeatureUnavailableContext("voice_mode");
+        if (unavailable) {
+          throw new VoiceFeatureUnavailableError(unavailable);
+        }
+
         const normalizedAgentId = this.parseVoiceTargetAgentId(
           agentId ?? "",
           "set_voice_mode"
@@ -1789,6 +1897,7 @@ export class Session {
     } catch (error) {
       const errorMessage =
         error instanceof Error ? error.message : "Failed to set voice mode";
+      const unavailable = this.getVoiceFeatureUnavailableResponseMetadata(error);
       this.sessionLogger.error(
         {
           err: error,
@@ -1806,6 +1915,7 @@ export class Session {
             agentId: this.voiceModeAgentId,
             accepted: false,
             error: errorMessage,
+            ...unavailable,
           },
         });
         return;

packages/server/src/server/speech/provider-resolver.ts

@@ -0,0 +1,8 @@
+export type Resolvable<T> = T | (() => T);
+
+export function toResolver<T>(value: Resolvable<T>): () => T {
+  if (typeof value === "function") {
+    return value as () => T;
+  }
+  return () => value;
+}

packages/server/src/server/speech/providers/local/runtime.ts

@@ -186,10 +186,12 @@ export async function initializeLocalSpeechServices(params: {
   providers: RequestedSpeechProviders;
   speechConfig: PaseoSpeechConfig | null;
   logger: Logger;
+  modelEnsureAutoDownload?: boolean;
 }): Promise<InitializedLocalSpeech> {
   const { providers, logger, speechConfig } = params;
   const localConfig = speechConfig?.local ?? null;
   const localModels = resolveConfiguredLocalModels(speechConfig);
+  const modelEnsureAutoDownload = params.modelEnsureAutoDownload ?? (localConfig?.autoDownload ?? true);
 
   let sttService: SpeechToTextProvider | null = null;
   let ttsService: TextToSpeechProvider | null = null;
@@ -207,14 +209,14 @@ export async function initializeLocalSpeechServices(params: {
         {
           modelsDir: localConfig.modelsDir,
           modelIds: requiredLocalModelIds,
-          autoDownload: localConfig.autoDownload ?? true,
+          autoDownload: modelEnsureAutoDownload,
         },
         "Ensuring local speech models"
       );
       await ensureLocalSpeechModels({
         modelsDir: localConfig.modelsDir,
         modelIds: requiredLocalModelIds,
-        autoDownload: localConfig.autoDownload ?? true,
+        autoDownload: modelEnsureAutoDownload,
         logger,
       });
     } catch (err) {
@@ -223,7 +225,7 @@ export async function initializeLocalSpeechServices(params: {
           err,
           modelsDir: localConfig.modelsDir,
           modelIds: requiredLocalModelIds,
-          autoDownload: localConfig.autoDownload ?? true,
+          autoDownload: modelEnsureAutoDownload,
           hint:
             "Use `paseo speech models` to inspect status and " +
             "`paseo speech download --model <MODEL_ID>` to fetch missing models.",

packages/server/src/server/speech/speech-runtime.test.ts

@@ -0,0 +1,140 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import pino from "pino";
+
+import type { PaseoSpeechConfig } from "../bootstrap.js";
+import type { InitializedLocalSpeech } from "./providers/local/runtime.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
+import { initializeSpeechRuntime } from "./speech-runtime.js";
+
+const { initializeLocalSpeechServicesMock } = vi.hoisted(() => ({
+  initializeLocalSpeechServicesMock: vi.fn<
+    (args: unknown) => Promise<InitializedLocalSpeech>
+  >(),
+}));
+
+vi.mock("./providers/local/runtime.js", () => ({
+  initializeLocalSpeechServices: initializeLocalSpeechServicesMock,
+}));
+
+vi.mock("./providers/openai/runtime.js", () => ({
+  getOpenAiSpeechAvailability: () => ({ configured: false }),
+  initializeOpenAiSpeechServices: (args: {
+    existing: {
+      sttService: SpeechToTextProvider | null;
+      ttsService: TextToSpeechProvider | null;
+      dictationSttService: SpeechToTextProvider | null;
+    };
+  }) => ({
+    sttService: args.existing.sttService,
+    ttsService: args.existing.ttsService,
+    dictationSttService: args.existing.dictationSttService,
+  }),
+  validateOpenAiCredentialRequirements: () => {},
+}));
+
+vi.mock("./providers/local/models.js", () => ({
+  ensureLocalSpeechModels: vi.fn(async () => {}),
+  getLocalSpeechModelDir: vi.fn(() => ""),
+  listLocalSpeechModels: vi.fn(() => []),
+}));
+
+function createStubStt(id: string): SpeechToTextProvider {
+  return {
+    id,
+    createSession: vi.fn(() => {
+      throw new Error("not used in this test");
+    }),
+  };
+}
+
+function createStubTts(id: string): TextToSpeechProvider {
+  return {
+    id,
+    synthesizeSpeech: vi.fn(async () => {
+      throw new Error("not used in this test");
+    }),
+  };
+}
+
+function createSpeechConfig(
+  providers: PaseoSpeechConfig["providers"]
+): PaseoSpeechConfig {
+  return { providers };
+}
+
+describe("initializeSpeechRuntime readiness", () => {
+  beforeEach(() => {
+    initializeLocalSpeechServicesMock.mockReset();
+  });
+
+  it("keeps voice feature available when only dictation is enabled and ready", async () => {
+    const dictationStt = createStubStt("dictation-local");
+
+    initializeLocalSpeechServicesMock.mockResolvedValue({
+      sttService: null,
+      ttsService: null,
+      dictationSttService: dictationStt,
+      localVoiceTtsProvider: null,
+      localModelConfig: null,
+      availability: {
+        configured: false,
+        modelsDir: null,
+        autoDownload: null,
+      },
+      cleanup: () => {},
+    });
+
+    const runtime = await initializeSpeechRuntime({
+      logger: pino({ level: "silent" }),
+      speechConfig: createSpeechConfig({
+        dictationStt: { provider: "local", enabled: true, explicit: true },
+        voiceStt: { provider: "local", enabled: false, explicit: true },
+        voiceTts: { provider: "local", enabled: false, explicit: true },
+      }),
+    });
+
+    const readiness = runtime.getSpeechReadiness();
+    expect(readiness.dictation.available).toBe(true);
+    expect(readiness.realtimeVoice.reasonCode).toBe("disabled");
+    expect(readiness.voiceFeature.available).toBe(true);
+    expect(readiness.voiceFeature.reasonCode).toBe("ready");
+
+    runtime.cleanup();
+  });
+
+  it("keeps voice feature available when only realtime voice is enabled and ready", async () => {
+    const voiceStt = createStubStt("voice-local");
+    const voiceTts = createStubTts("tts-local");
+
+    initializeLocalSpeechServicesMock.mockResolvedValue({
+      sttService: voiceStt,
+      ttsService: voiceTts,
+      dictationSttService: null,
+      localVoiceTtsProvider: voiceTts,
+      localModelConfig: null,
+      availability: {
+        configured: false,
+        modelsDir: null,
+        autoDownload: null,
+      },
+      cleanup: () => {},
+    });
+
+    const runtime = await initializeSpeechRuntime({
+      logger: pino({ level: "silent" }),
+      speechConfig: createSpeechConfig({
+        dictationStt: { provider: "local", enabled: false, explicit: true },
+        voiceStt: { provider: "local", enabled: true, explicit: true },
+        voiceTts: { provider: "local", enabled: true, explicit: true },
+      }),
+    });
+
+    const readiness = runtime.getSpeechReadiness();
+    expect(readiness.realtimeVoice.available).toBe(true);
+    expect(readiness.dictation.reasonCode).toBe("disabled");
+    expect(readiness.voiceFeature.available).toBe(true);
+    expect(readiness.voiceFeature.reasonCode).toBe("ready");
+
+    runtime.cleanup();
+  });
+});

packages/server/src/server/speech/speech-runtime.ts

@@ -1,11 +1,15 @@
+import { stat } from "node:fs/promises";
+import { join } from "node:path";
 import type { Logger } from "pino";
 
 import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
-import {
-  getLocalSpeechAvailability,
-  initializeLocalSpeechServices,
-} from "./providers/local/runtime.js";
 import type { LocalSpeechModelId } from "./providers/local/config.js";
+import {
+  ensureLocalSpeechModels,
+  getLocalSpeechModelDir,
+  listLocalSpeechModels,
+} from "./providers/local/models.js";
+import { initializeLocalSpeechServices } from "./providers/local/runtime.js";
 import {
   getOpenAiSpeechAvailability,
   initializeOpenAiSpeechServices,
@@ -14,6 +18,39 @@ import {
 import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
 import type { RequestedSpeechProviders } from "./speech-types.js";
 
+const SPEECH_RUNTIME_MONITOR_INTERVAL_MS = 3000;
+
+export type SpeechReadinessReasonCode =
+  | "ready"
+  | "disabled"
+  | "model_download_in_progress"
+  | "models_missing"
+  | "model_download_failed"
+  | "stt_unavailable"
+  | "tts_unavailable";
+
+export type SpeechReadinessState = {
+  enabled: boolean;
+  available: boolean;
+  reasonCode: SpeechReadinessReasonCode;
+  message: string;
+  retryable: boolean;
+  missingModelIds: LocalSpeechModelId[];
+};
+
+export type SpeechReadinessSnapshot = {
+  generatedAt: string;
+  requiredLocalModelIds: LocalSpeechModelId[];
+  missingLocalModelIds: LocalSpeechModelId[];
+  download: {
+    inProgress: boolean;
+    error: string | null;
+  };
+  realtimeVoice: SpeechReadinessState;
+  dictation: SpeechReadinessState;
+  voiceFeature: SpeechReadinessState;
+};
+
 function resolveRequestedSpeechProviders(
   speechConfig: PaseoSpeechConfig | null
 ): RequestedSpeechProviders {
@@ -29,10 +66,248 @@ function resolveRequestedSpeechProviders(
   };
 }
 
-export type InitializedSpeechRuntime = {
+async function hasRequiredLocalModelFile(filePath: string): Promise<boolean> {
+  try {
+    const fileStat = await stat(filePath);
+    if (fileStat.isDirectory()) {
+      return true;
+    }
+    return fileStat.isFile() && fileStat.size > 0;
+  } catch {
+    return false;
+  }
+}
+
+async function findMissingRequiredLocalModels(params: {
+  modelsDir: string | null;
+  requiredModelIds: LocalSpeechModelId[];
+}): Promise<LocalSpeechModelId[]> {
+  const { modelsDir, requiredModelIds } = params;
+  if (!modelsDir || requiredModelIds.length === 0) {
+    return [];
+  }
+
+  const specsById = new Map(listLocalSpeechModels().map((model) => [model.id, model]));
+  const missing = new Set<LocalSpeechModelId>();
+
+  for (const modelId of requiredModelIds) {
+    const spec = specsById.get(modelId);
+    if (!spec) {
+      missing.add(modelId);
+      continue;
+    }
+    const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
+    for (const relPath of spec.requiredFiles) {
+      const filePath = join(modelDir, relPath);
+      if (!(await hasRequiredLocalModelFile(filePath))) {
+        missing.add(modelId);
+        break;
+      }
+    }
+  }
+
+  return Array.from(missing);
+}
+
+function joinModelIds(modelIds: LocalSpeechModelId[]): string {
+  if (modelIds.length === 0) {
+    return "none";
+  }
+  return modelIds.join(", ");
+}
+
+function buildRealtimeVoiceReadiness(params: {
+  providers: RequestedSpeechProviders;
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+}): SpeechReadinessState {
+  const voiceSttEnabled = params.providers.voiceStt.enabled !== false;
+  const voiceTtsEnabled = params.providers.voiceTts.enabled !== false;
+  const enabled = voiceSttEnabled || voiceTtsEnabled;
+  if (!enabled) {
+    return {
+      enabled: false,
+      available: false,
+      reasonCode: "disabled",
+      message: "Realtime voice is disabled in daemon config.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  if (voiceSttEnabled && !params.sttService) {
+    return {
+      enabled: true,
+      available: false,
+      reasonCode: "stt_unavailable",
+      message: "Realtime voice is unavailable: speech-to-text service is not ready.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  if (voiceTtsEnabled && !params.ttsService) {
+    return {
+      enabled: true,
+      available: false,
+      reasonCode: "tts_unavailable",
+      message: "Realtime voice is unavailable: text-to-speech service is not ready.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  return {
+    enabled: true,
+    available: true,
+    reasonCode: "ready",
+    message: "Realtime voice is ready.",
+    retryable: false,
+    missingModelIds: [],
+  };
+}
+
+function buildDictationReadiness(params: {
+  providers: RequestedSpeechProviders;
+  dictationSttService: SpeechToTextProvider | null;
+}): SpeechReadinessState {
+  const enabled = params.providers.dictationStt.enabled !== false;
+  if (!enabled) {
+    return {
+      enabled: false,
+      available: false,
+      reasonCode: "disabled",
+      message: "Dictation is disabled in daemon config.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  if (!params.dictationSttService) {
+    return {
+      enabled: true,
+      available: false,
+      reasonCode: "stt_unavailable",
+      message: "Dictation is unavailable: speech-to-text service is not ready.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  return {
+    enabled: true,
+    available: true,
+    reasonCode: "ready",
+    message: "Dictation is ready.",
+    retryable: false,
+    missingModelIds: [],
+  };
+}
+
+function buildVoiceFeatureReadiness(params: {
+  realtimeVoice: SpeechReadinessState;
+  dictation: SpeechReadinessState;
+  missingLocalModelIds: LocalSpeechModelId[];
+  backgroundDownloadInProgress: boolean;
+  backgroundDownloadError: string | null;
+  localAutoDownloadEnabled: boolean;
+}): SpeechReadinessState {
+  const enabled = params.realtimeVoice.enabled || params.dictation.enabled;
+  if (!enabled) {
+    return {
+      enabled: false,
+      available: false,
+      reasonCode: "disabled",
+      message: "Voice features are disabled in daemon config.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+
+  if (params.missingLocalModelIds.length > 0) {
+    const missingModelIds = [...params.missingLocalModelIds];
+    if (params.backgroundDownloadInProgress) {
+      return {
+        enabled: true,
+        available: false,
+        reasonCode: "model_download_in_progress",
+        message: `Voice features are unavailable while models download in the background (${joinModelIds(missingModelIds)}).`,
+        retryable: true,
+        missingModelIds,
+      };
+    }
+    if (params.backgroundDownloadError) {
+      return {
+        enabled: true,
+        available: false,
+        reasonCode: "model_download_failed",
+        message: `Voice features are unavailable: model download failed (${params.backgroundDownloadError}).`,
+        retryable: false,
+        missingModelIds,
+      };
+    }
+    return {
+      enabled: true,
+      available: false,
+      reasonCode: "models_missing",
+      message: `Voice features are unavailable: missing local models (${joinModelIds(missingModelIds)}).`,
+      retryable: params.localAutoDownloadEnabled,
+      missingModelIds,
+    };
+  }
+
+  return {
+    enabled: true,
+    available: true,
+    reasonCode: "ready",
+    message: "Voice features are ready.",
+    retryable: false,
+    missingModelIds: [],
+  };
+}
+
+function describeRequestedProviders(providers: RequestedSpeechProviders): {
+  dictationStt: { provider: string; enabled: boolean; explicit: boolean };
+  voiceStt: { provider: string; enabled: boolean; explicit: boolean };
+  voiceTts: { provider: string; enabled: boolean; explicit: boolean };
+} {
+  return {
+    dictationStt: {
+      provider: providers.dictationStt.provider,
+      enabled: providers.dictationStt.enabled !== false,
+      explicit: providers.dictationStt.explicit,
+    },
+    voiceStt: {
+      provider: providers.voiceStt.provider,
+      enabled: providers.voiceStt.enabled !== false,
+      explicit: providers.voiceStt.explicit,
+    },
+    voiceTts: {
+      provider: providers.voiceTts.provider,
+      enabled: providers.voiceTts.enabled !== false,
+      explicit: providers.voiceTts.explicit,
+    },
+  };
+}
+
+function resolveEffectiveProviderIds(params: {
   sttService: SpeechToTextProvider | null;
   ttsService: TextToSpeechProvider | null;
   dictationSttService: SpeechToTextProvider | null;
+  localVoiceTtsProvider: TextToSpeechProvider | null;
+}): { dictationStt: string; voiceStt: string; voiceTts: string } {
+  return {
+    dictationStt: params.dictationSttService?.id ?? "unavailable",
+    voiceStt: params.sttService?.id ?? "unavailable",
+    voiceTts:
+      !params.ttsService
+        ? "unavailable"
+        : params.ttsService === params.localVoiceTtsProvider
+          ? "local"
+          : "openai",
+  };
+}
+
+export type InitializedSpeechRuntime = {
+  resolveVoiceStt: () => SpeechToTextProvider | null;
+  resolveVoiceTts: () => TextToSpeechProvider | null;
+  resolveDictationStt: () => SpeechToTextProvider | null;
+  getSpeechReadiness: () => SpeechReadinessSnapshot;
   cleanup: () => void;
   localModelConfig: {
     modelsDir: string;
@@ -45,10 +320,12 @@ export async function initializeSpeechRuntime(params: {
   openaiConfig?: PaseoOpenAIConfig;
   speechConfig?: PaseoSpeechConfig;
 }): Promise<InitializedSpeechRuntime> {
-  const logger = params.logger;
+  const logger = params.logger.child({ module: "speech-runtime" });
   const speechConfig = params.speechConfig ?? null;
   const openaiConfig = params.openaiConfig;
   const providers = resolveRequestedSpeechProviders(speechConfig);
+  const requestedProviders = describeRequestedProviders(providers);
+  const localAutoDownloadEnabled = speechConfig?.local?.autoDownload ?? true;
 
   validateOpenAiCredentialRequirements({
     providers,
@@ -58,145 +335,264 @@ export async function initializeSpeechRuntime(params: {
 
   logger.info(
     {
-      requestedProviders: {
+      requestedProviders,
-        dictationStt: {
-          provider: providers.dictationStt.provider,
-          enabled: providers.dictationStt.enabled !== false,
-        },
-        voiceStt: {
-          provider: providers.voiceStt.provider,
-          enabled: providers.voiceStt.enabled !== false,
-        },
-        voiceTts: {
-          provider: providers.voiceTts.provider,
-          enabled: providers.voiceTts.enabled !== false,
-        },
-      },
       availability: {
         openai: getOpenAiSpeechAvailability(openaiConfig),
-        local: getLocalSpeechAvailability(speechConfig),
       },
     },
     "Speech provider reconciliation started"
   );
 
-  const localSpeech = await initializeLocalSpeechServices({
+  let sttService: SpeechToTextProvider | null = null;
-    providers,
+  let ttsService: TextToSpeechProvider | null = null;
-    speechConfig,
+  let dictationSttService: SpeechToTextProvider | null = null;
-    logger,
+  let localModelConfig: {
-  });
+    modelsDir: string;
+    defaultModelIds: LocalSpeechModelId[];
+  } | null = null;
+  let localCleanup = () => {};
+  let localVoiceTtsProvider: TextToSpeechProvider | null = null;
 
-  const openAiSpeech = initializeOpenAiSpeechServices({
+  let missingLocalModelIds: LocalSpeechModelId[] = [];
-    providers,
+  let backgroundDownloadInProgress = false;
-    openaiConfig,
+  let backgroundDownloadError: string | null = null;
-    existing: {
+  let stopped = false;
-      sttService: localSpeech.sttService,
+  let monitorTimeout: ReturnType<typeof setTimeout> | null = null;
-      ttsService: localSpeech.ttsService,
+  let reconcileInFlight: Promise<void> | null = null;
-      dictationSttService: localSpeech.dictationSttService,
-    },
-    logger,
-  });
 
-  const effectiveProviders = {
+  const computeReadinessSnapshot = (): SpeechReadinessSnapshot => {
-    dictationStt: openAiSpeech.dictationSttService?.id ?? "unavailable",
+    const realtimeVoice = buildRealtimeVoiceReadiness({
-    voiceStt: openAiSpeech.sttService?.id ?? "unavailable",
+      providers,
-    voiceTts:
+      sttService,
-      !openAiSpeech.ttsService
+      ttsService,
-        ? "unavailable"
+    });
-        : openAiSpeech.ttsService === localSpeech.localVoiceTtsProvider
+    const dictation = buildDictationReadiness({
-          ? "local"
+      providers,
-          : "openai",
+      dictationSttService,
+    });
+    const voiceFeature = buildVoiceFeatureReadiness({
+      realtimeVoice,
+      dictation,
+      missingLocalModelIds,
+      backgroundDownloadInProgress,
+      backgroundDownloadError,
+      localAutoDownloadEnabled,
+    });
+    return {
+      generatedAt: new Date().toISOString(),
+      requiredLocalModelIds: localModelConfig?.defaultModelIds ?? [],
+      missingLocalModelIds: [...missingLocalModelIds],
+      download: {
+        inProgress: backgroundDownloadInProgress,
+        error: backgroundDownloadError,
+      },
+      realtimeVoice: {
+        ...realtimeVoice,
+      },
+      dictation: {
+        ...dictation,
+      },
+      voiceFeature: {
+        ...voiceFeature,
+      },
+    };
   };
-  const unavailableFeatures = [
-    providers.dictationStt.enabled !== false && !openAiSpeech.dictationSttService
-      ? "dictation.stt"
-      : null,
-    providers.voiceStt.enabled !== false && !openAiSpeech.sttService ? "voice.stt" : null,
-    providers.voiceTts.enabled !== false && !openAiSpeech.ttsService ? "voice.tts" : null,
-  ].filter((feature): feature is string => feature !== null);
-  const explicitlyConfiguredUnavailableFeatures = unavailableFeatures.filter((feature) => {
-    if (feature === "dictation.stt") {
-      return providers.dictationStt.explicit;
-    }
-    if (feature === "voice.stt") {
-      return providers.voiceStt.explicit;
-    }
-    return providers.voiceTts.explicit;
-  });
 
-  if (explicitlyConfiguredUnavailableFeatures.length > 0) {
+  const refreshMissingLocalModels = async (): Promise<void> => {
-    logger.error(
+    missingLocalModelIds = await findMissingRequiredLocalModels({
-      {
+      modelsDir: localModelConfig?.modelsDir ?? null,
-        requestedProviders: {
+      requiredModelIds: localModelConfig?.defaultModelIds ?? [],
-          dictationStt: {
+    });
-            provider: providers.dictationStt.provider,
+  };
-            enabled: providers.dictationStt.enabled !== false,
-          },
-          voiceStt: {
-            provider: providers.voiceStt.provider,
-            enabled: providers.voiceStt.enabled !== false,
-          },
-          voiceTts: {
-            provider: providers.voiceTts.provider,
-            enabled: providers.voiceTts.enabled !== false,
-          },
-        },
-        explicitProviders: {
-          dictationStt: providers.dictationStt.explicit,
-          voiceStt: providers.voiceStt.explicit,
-          voiceTts: providers.voiceTts.explicit,
-        },
-        effectiveProviders,
-        unavailableFeatures: explicitlyConfiguredUnavailableFeatures,
-      },
-      "Speech provider reconciliation failed: configured features are unavailable"
-    );
-    throw new Error(
-      `Configured speech features unavailable: ${explicitlyConfiguredUnavailableFeatures.join(", ")}`
-    );
-  }
 
-  if (unavailableFeatures.length > 0) {
+  const reconcileServices = async (modelEnsureAutoDownload: boolean): Promise<void> => {
-    logger.warn(
+    const nextLocalSpeech = await initializeLocalSpeechServices({
-      {
+      providers,
-        requestedProviders: {
+      speechConfig,
-          dictationStt: {
+      logger,
-            provider: providers.dictationStt.provider,
+      modelEnsureAutoDownload,
-            enabled: providers.dictationStt.enabled !== false,
+    });
-          },
+    const nextOpenAiSpeech = initializeOpenAiSpeechServices({
-          voiceStt: {
+      providers,
-            provider: providers.voiceStt.provider,
+      openaiConfig,
-            enabled: providers.voiceStt.enabled !== false,
+      existing: {
-          },
+        sttService: nextLocalSpeech.sttService,
-          voiceTts: {
+        ttsService: nextLocalSpeech.ttsService,
-            provider: providers.voiceTts.provider,
+        dictationSttService: nextLocalSpeech.dictationSttService,
-            enabled: providers.voiceTts.enabled !== false,
-          },
-        },
-        explicitProviders: {
-          dictationStt: providers.dictationStt.explicit,
-          voiceStt: providers.voiceStt.explicit,
-          voiceTts: providers.voiceTts.explicit,
-        },
-        effectiveProviders,
-        unavailableFeatures,
       },
-      "Speech provider reconciliation completed with unavailable default features"
+      logger,
-    );
+    });
-  } else {
+
+    const previousLocalCleanup = localCleanup;
+    sttService = nextOpenAiSpeech.sttService;
+    ttsService = nextOpenAiSpeech.ttsService;
+    dictationSttService = nextOpenAiSpeech.dictationSttService;
+    localModelConfig = nextLocalSpeech.localModelConfig;
+    localVoiceTtsProvider = nextLocalSpeech.localVoiceTtsProvider;
+    localCleanup = nextLocalSpeech.cleanup;
+    previousLocalCleanup();
+
+    await refreshMissingLocalModels();
+
+    const effectiveProviders = resolveEffectiveProviderIds({
+      sttService,
+      ttsService,
+      dictationSttService,
+      localVoiceTtsProvider,
+    });
+    const unavailableFeatures = [
+      providers.dictationStt.enabled !== false && !dictationSttService ? "dictation.stt" : null,
+      providers.voiceStt.enabled !== false && !sttService ? "voice.stt" : null,
+      providers.voiceTts.enabled !== false && !ttsService ? "voice.tts" : null,
+    ].filter((feature): feature is string => feature !== null);
+
+    if (unavailableFeatures.length > 0) {
+      logger.warn(
+        {
+          requestedProviders,
+          effectiveProviders,
+          unavailableFeatures,
+          missingLocalModelIds,
+        },
+        "Speech provider reconciliation completed with unavailable features"
+      );
+    } else {
+      logger.info(
+        {
+          requestedProviders,
+          effectiveProviders,
+        },
+        "Speech provider reconciliation completed"
+      );
+    }
+  };
+
+  const runReconcile = async (params: { modelEnsureAutoDownload: boolean }): Promise<void> => {
+    if (reconcileInFlight) {
+      await reconcileInFlight;
+      return;
+    }
+    reconcileInFlight = reconcileServices(params.modelEnsureAutoDownload).finally(() => {
+      reconcileInFlight = null;
+    });
+    await reconcileInFlight;
+  };
+
+  const scheduleMonitor = (): void => {
+    if (stopped || monitorTimeout) {
+      return;
+    }
+    monitorTimeout = setTimeout(() => {
+      monitorTimeout = null;
+      void runMonitorTick();
+    }, SPEECH_RUNTIME_MONITOR_INTERVAL_MS);
+  };
+
+  const startBackgroundDownload = (): void => {
+    if (stopped || backgroundDownloadInProgress) {
+      return;
+    }
+    const modelsDir = localModelConfig?.modelsDir ?? null;
+    const modelIds = [...missingLocalModelIds];
+    if (!modelsDir || modelIds.length === 0 || !localAutoDownloadEnabled) {
+      return;
+    }
+
+    backgroundDownloadInProgress = true;
+    backgroundDownloadError = null;
+
     logger.info(
       {
-        effectiveProviders,
+        modelsDir,
+        modelIds,
       },
-      "Speech provider reconciliation completed"
+      "Starting background download for missing local speech models"
     );
+
+    void (async () => {
+      try {
+        await ensureLocalSpeechModels({
+          modelsDir,
+          modelIds,
+          autoDownload: true,
+          logger,
+        });
+        await runReconcile({ modelEnsureAutoDownload: false });
+        backgroundDownloadError = null;
+      } catch (error) {
+        backgroundDownloadError = error instanceof Error ? error.message : String(error);
+        logger.error(
+          {
+            err: error,
+            modelIds,
+          },
+          "Background local speech model download failed"
+        );
+      } finally {
+        backgroundDownloadInProgress = false;
+        await refreshMissingLocalModels().catch((error) => {
+          logger.warn({ err: error }, "Failed to refresh local speech model status after download");
+        });
+        scheduleMonitor();
+      }
+    })();
+  };
+
+  const runMonitorTick = async (): Promise<void> => {
+    if (stopped) {
+      return;
+    }
+    try {
+      await refreshMissingLocalModels();
+      const snapshot = computeReadinessSnapshot();
+      if (
+        snapshot.voiceFeature.enabled &&
+        !snapshot.voiceFeature.available &&
+        missingLocalModelIds.length === 0 &&
+        !backgroundDownloadInProgress
+      ) {
+        await runReconcile({ modelEnsureAutoDownload: false });
+      }
+
+      if (
+        missingLocalModelIds.length > 0 &&
+        localAutoDownloadEnabled &&
+        !backgroundDownloadInProgress &&
+        !backgroundDownloadError
+      ) {
+        startBackgroundDownload();
+      }
+    } catch (error) {
+      logger.warn({ err: error }, "Speech runtime monitor tick failed");
+    } finally {
+      scheduleMonitor();
+    }
+  };
+
+  await runReconcile({ modelEnsureAutoDownload: false });
+  const snapshot = computeReadinessSnapshot();
+  if (snapshot.voiceFeature.enabled && !snapshot.voiceFeature.available) {
+    if (missingLocalModelIds.length > 0 && localAutoDownloadEnabled) {
+      startBackgroundDownload();
+    }
+    scheduleMonitor();
   }
 
+  const cleanup = (): void => {
+    stopped = true;
+    if (monitorTimeout) {
+      clearTimeout(monitorTimeout);
+      monitorTimeout = null;
+    }
+    localCleanup();
+  };
+
   return {
-    sttService: openAiSpeech.sttService,
+    resolveVoiceStt: () => sttService,
-    ttsService: openAiSpeech.ttsService,
+    resolveVoiceTts: () => ttsService,
-    dictationSttService: openAiSpeech.dictationSttService,
+    resolveDictationStt: () => dictationSttService,
-    cleanup: localSpeech.cleanup,
+    getSpeechReadiness: () => computeReadinessSnapshot(),
-    localModelConfig: localSpeech.localModelConfig,
+    cleanup,
+    localModelConfig,
   };
 }

packages/server/src/server/websocket-server.ts

@@ -26,6 +26,8 @@ import type { AgentProviderRuntimeSettingsMap } from "./agent/provider-launch-co
 import { PushTokenStore } from "./push/token-store.js";
 import { PushService } from "./push/push-service.js";
 import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
+import type { Resolvable } from "./speech/provider-resolver.js";
+import type { SpeechReadinessSnapshot } from "./speech/speech-runtime.js";
 import type { LocalSpeechModelId } from "./speech/providers/local/models.js";
 import type {
   VoiceCallerContext,
@@ -93,16 +95,17 @@ export class VoiceAssistantWebSocketServer {
   private readonly pushTokenStore: PushTokenStore;
   private readonly pushService: PushService;
   private readonly createAgentMcpTransport: AgentMcpTransportFactory;
-  private readonly stt: SpeechToTextProvider | null;
+  private readonly stt: Resolvable<SpeechToTextProvider | null>;
-  private readonly tts: TextToSpeechProvider | null;
+  private readonly tts: Resolvable<TextToSpeechProvider | null>;
   private readonly terminalManager: TerminalManager | null;
   private readonly dictation: {
     finalTimeoutMs?: number;
-    stt?: SpeechToTextProvider | null;
+    stt?: Resolvable<SpeechToTextProvider | null>;
     localModels?: {
       modelsDir: string;
       defaultModelIds: LocalSpeechModelId[];
     };
+    getSpeechReadiness?: () => SpeechReadinessSnapshot;
   } | null;
   private readonly voice: {
     voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
@@ -126,7 +129,10 @@ export class VoiceAssistantWebSocketServer {
     paseoHome: string,
     createAgentMcpTransport: AgentMcpTransportFactory,
     wsConfig: WebSocketServerConfig,
-    speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
+    speech?: {
+      stt: Resolvable<SpeechToTextProvider | null>;
+      tts: Resolvable<TextToSpeechProvider | null>;
+    },
     terminalManager?: TerminalManager | null,
     voice?: {
       voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
@@ -135,11 +141,12 @@ export class VoiceAssistantWebSocketServer {
     },
     dictation?: {
       finalTimeoutMs?: number;
-      stt?: SpeechToTextProvider | null;
+      stt?: Resolvable<SpeechToTextProvider | null>;
       localModels?: {
         modelsDir: string;
         defaultModelIds: LocalSpeechModelId[];
       };
+      getSpeechReadiness?: () => SpeechReadinessSnapshot;
     },
     agentProviderRuntimeSettings?: AgentProviderRuntimeSettingsMap
   ) {

packages/server/src/shared/messages.ts

@@ -691,6 +691,9 @@ export const SetVoiceModeResponseMessageSchema = z.object({
     agentId: z.string().nullable(),
     accepted: z.boolean(),
     error: z.string().nullable(),
+    reasonCode: z.string().optional(),
+    retryable: z.boolean().optional(),
+    missingModelIds: z.array(z.string()).optional(),
   }),
 });
 
@@ -1159,6 +1162,8 @@ export const DictationStreamErrorMessageSchema = z.object({
     dictationId: z.string(),
     error: z.string(),
     retryable: z.boolean(),
+    reasonCode: z.string().optional(),
+    missingModelIds: z.array(z.string()).optional(),
     debugRecordingPath: z.string().optional(),
   }),
 });