Refactor speech provider selection model and harden CI startup

packages/server/src/server/bootstrap.smoke.test.ts

@@ -13,9 +13,16 @@ describe("paseo daemon bootstrap", () => {
     const daemonHandle = await createTestPaseoDaemon({
       openai: { apiKey: "test-openai-api-key" },
       speech: {
-        dictationSttProvider: "openai",
+        providers: {
-        voiceSttProvider: "openai",
+          dictationStt: { provider: "openai", explicit: true },
-        voiceTtsProvider: "openai",
+          voiceStt: { provider: "openai", explicit: true },
+          voiceTts: { provider: "openai", explicit: true },
+        },
+        localModels: {
+          dictationStt: "parakeet-tdt-0.6b-v3-int8",
+          voiceStt: "parakeet-tdt-0.6b-v3-int8",
+          voiceTts: "pocket-tts-onnx-int8",
+        },
       },
     });
     try {
@@ -56,9 +63,16 @@ describe("paseo daemon bootstrap", () => {
       appBaseUrl: "https://app.paseo.sh",
       openai: undefined,
       speech: {
-        dictationSttProvider: "openai",
+        providers: {
-        voiceSttProvider: "openai",
+          dictationStt: { provider: "openai", explicit: true },
-        voiceTtsProvider: "openai",
+          voiceStt: { provider: "openai", explicit: true },
+          voiceTts: { provider: "openai", explicit: true },
+        },
+        localModels: {
+          dictationStt: "parakeet-tdt-0.6b-v3-int8",
+          voiceStt: "parakeet-tdt-0.6b-v3-int8",
+          voiceTts: "pocket-tts-onnx-int8",
+        },
       },
     };
 

packages/server/src/server/bootstrap.ts

@@ -43,6 +43,7 @@ import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
 import { DownloadTokenStore } from "./file-download/token-store.js";
 import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js";
 import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js";
+import type { RequestedSpeechProviders } from "./speech/speech-types.js";
 import { initializeSpeechRuntime } from "./speech/speech-runtime.js";
 import { AgentManager } from "./agent/agent-manager.js";
 import { AgentStorage } from "./agent/agent-storage.js";
@@ -105,15 +106,15 @@ export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig;
 export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig;
 
 export type PaseoSpeechConfig = {
-  dictationSttProvider?: "openai" | "local";
+  providers: RequestedSpeechProviders;
-  voiceSttProvider?: "openai" | "local";
-  voiceTtsProvider?: "openai" | "local";
   local?: PaseoLocalSpeechConfig;
-  dictationLocalSttModel?: string;
+  localModels: {
-  voiceLocalSttModel?: string;
+    dictationStt: string;
-  voiceLocalTtsModel?: string;
+    voiceStt: string;
-  voiceLocalTtsSpeakerId?: number;
+    voiceTts: string;
-  voiceLocalTtsSpeed?: number;
+    voiceTtsSpeakerId?: number;
+    voiceTtsSpeed?: number;
+  };
 };
 
 export type PaseoDaemonConfig = {
@@ -347,25 +348,34 @@ export async function createPaseoDaemon(
       voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null;
   }
 
-  if (!resolvedVoiceLlmProvider) {
+  let resolvedVoiceLlmModeId: string | null = null;
-    logger.error(
+  let resolvedVoiceLlmModel: string | null = null;
-      { requestedVoiceLlmProvider, voiceLlmAvailability },
-      "No voice LLM provider available"
-    );
-    throw new Error("No voice LLM provider available");
-  }
 
-  const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
+  if (!resolvedVoiceLlmProvider) {
-    (definition) => definition.id === resolvedVoiceLlmProvider
+    if (voiceLlmProviderExplicit) {
-  );
+      logger.error(
-  if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
+        { requestedVoiceLlmProvider, voiceLlmAvailability },
-    throw new Error(
+        "No voice LLM provider available"
-      `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
+      );
+      throw new Error("No voice LLM provider available");
+    }
+    logger.warn(
+      { requestedVoiceLlmProvider, voiceLlmAvailability },
+      "No default voice LLM provider available; voice mode will be disabled until a provider is configured"
     );
+  } else {
+    const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
+      (definition) => definition.id === resolvedVoiceLlmProvider
+    );
+    if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
+      throw new Error(
+        `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
+      );
+    }
+    resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
+    resolvedVoiceLlmModel =
+      config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;
   }
-  const resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
-  const resolvedVoiceLlmModel =
-    config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;
 
   logger.info(
     {

packages/server/src/server/daemon-client.e2e.test.ts

@@ -93,33 +93,47 @@ describe("daemon client E2E", () => {
   let ctx: DaemonTestContext;
 
   beforeAll(async () => {
-    const speechConfig = openaiApiKey
+    const speechConfig =
-      ? {
+      openaiApiKey
-          dictationSttProvider: "openai" as const,
+        ? {
-          voiceSttProvider: "openai" as const,
+            providers: {
-          voiceTtsProvider: "openai" as const,
+              dictationStt: { provider: "openai" as const, explicit: true },
-        }
+              voiceStt: { provider: "openai" as const, explicit: true },
-      : {
+              voiceTts: { provider: "openai" as const, explicit: true },
-          dictationSttProvider: "local" as const,
+            },
-          voiceSttProvider: "local" as const,
+            localModels: {
-          voiceTtsProvider: "local" as const,
+              dictationStt: "parakeet-tdt-0.6b-v3-int8",
-          local: {
+              voiceStt: "parakeet-tdt-0.6b-v3-int8",
-            modelsDir: localModelsDir,
+              voiceTts: "pocket-tts-onnx-int8",
-          },
+            },
-          dictationLocalSttModel:
+          }
-            process.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
+        : hasLocalSpeech
-            "zipformer-bilingual-zh-en-2023-02-20",
+          ? {
-          voiceLocalSttModel:
+              providers: {
-            process.env.PASEO_VOICE_LOCAL_STT_MODEL ??
+                dictationStt: { provider: "local" as const, explicit: true },
-            "zipformer-bilingual-zh-en-2023-02-20",
+                voiceStt: { provider: "local" as const, explicit: true },
-          voiceLocalTtsModel:
+                voiceTts: { provider: "local" as const, explicit: true },
-            process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
+              },
-        };
+              local: {
+                modelsDir: localModelsDir,
+              },
+              localModels: {
+                dictationStt:
+                  process.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
+                  "zipformer-bilingual-zh-en-2023-02-20",
+                voiceStt:
+                  process.env.PASEO_VOICE_LOCAL_STT_MODEL ??
+                  "zipformer-bilingual-zh-en-2023-02-20",
+                voiceTts:
+                  process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
+              },
+            }
+          : undefined;
 
     ctx = await createDaemonTestContext({
       dictationFinalTimeoutMs: 5000,
       ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}),
-      speech: speechConfig,
+      ...(speechConfig ? { speech: speechConfig } : {}),
     });
   }, 60000);
 

packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts

@@ -186,6 +186,7 @@ describe("ConnectionOfferV2 (daemon E2E)", () => {
         PASEO_HOME: tempHome,
         PASEO_LISTEN: `0.0.0.0:${port}`,
         OPENAI_API_KEY: "",
+        PASEO_LOCAL_AUTO_DOWNLOAD: "0",
         PASEO_LOG_FORMAT: "json",
       };
 

packages/server/src/server/persisted-config.ts

@@ -32,15 +32,11 @@ const ProvidersSchema = z
   })
   .strict();
 
-const SpeechProviderIdSchema = z.preprocess(
+const SpeechProviderIdSchema = z
-  (value) => {
+  .string()
-    if (typeof value !== "string") {
+  .trim()
-      return value;
+  .toLowerCase()
-    }
+  .pipe(z.enum(["openai", "local"]));
-    return value.trim().toLowerCase();
-  },
-  z.enum(["openai", "local"])
-);
 
 const FeatureDictationSchema = z
   .object({

packages/server/src/server/speech/providers/local/config.ts

@@ -68,7 +68,7 @@ const LocalSpeechResolutionSchema = z.object({
 });
 
 function persistedLocalFeatureModel(
-  provider: RequestedSpeechProviders[keyof RequestedSpeechProviders],
+  provider: RequestedSpeechProviders[keyof RequestedSpeechProviders]["provider"],
   model: string | undefined
 ): string | undefined {
   if (provider !== "local") {
@@ -83,9 +83,9 @@ function shouldIncludeLocalProviderConfig(params: {
   persisted: PersistedConfig;
 }): boolean {
   const localRequestedByFeature =
-    params.providers.dictationSttProvider === "local" ||
+    params.providers.dictationStt.provider === "local" ||
-    params.providers.voiceSttProvider === "local" ||
+    params.providers.voiceStt.provider === "local" ||
-    params.providers.voiceTtsProvider === "local";
+    params.providers.voiceTts.provider === "local";
 
   return (
     localRequestedByFeature ||
@@ -114,21 +114,21 @@ export function resolveLocalSpeechConfig(params: {
     dictationLocalSttModel:
       params.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
       persistedLocalFeatureModel(
-        params.providers.dictationSttProvider,
+        params.providers.dictationStt.provider,
         params.persisted.features?.dictation?.stt?.model
       ) ??
       DEFAULT_LOCAL_STT_MODEL,
     voiceLocalSttModel:
       params.env.PASEO_VOICE_LOCAL_STT_MODEL ??
       persistedLocalFeatureModel(
-        params.providers.voiceSttProvider,
+        params.providers.voiceStt.provider,
         params.persisted.features?.voiceMode?.stt?.model
       ) ??
       DEFAULT_LOCAL_STT_MODEL,
     voiceLocalTtsModel:
       params.env.PASEO_VOICE_LOCAL_TTS_MODEL ??
       persistedLocalFeatureModel(
-        params.providers.voiceTtsProvider,
+        params.providers.voiceTts.provider,
         params.persisted.features?.voiceMode?.tts?.model
       ) ??
       DEFAULT_LOCAL_TTS_MODEL,

packages/server/src/server/speech/providers/local/runtime.ts

@@ -63,13 +63,13 @@ function resolveConfiguredLocalModels(
 ): ResolvedLocalModels {
   return {
     dictationLocalSttModel: LocalSttModelIdSchema.parse(
-      speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
+      speechConfig?.localModels.dictationStt ?? DEFAULT_LOCAL_STT_MODEL
     ),
     voiceLocalSttModel: LocalSttModelIdSchema.parse(
-      speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
+      speechConfig?.localModels.voiceStt ?? DEFAULT_LOCAL_STT_MODEL
     ),
     voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
-      speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL
+      speechConfig?.localModels.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL
     ),
   };
 }
@@ -90,13 +90,13 @@ function computeRequiredLocalModelIds(params: {
   models: ResolvedLocalModels;
 }): LocalSpeechModelId[] {
   const ids = new Set<LocalSpeechModelId>();
-  if (params.providers.dictationSttProvider === "local") {
+  if (params.providers.dictationStt.provider === "local") {
     ids.add(params.models.dictationLocalSttModel);
   }
-  if (params.providers.voiceSttProvider === "local") {
+  if (params.providers.voiceStt.provider === "local") {
     ids.add(params.models.voiceLocalSttModel);
   }
-  if (params.providers.voiceTtsProvider === "local") {
+  if (params.providers.voiceTts.provider === "local") {
     ids.add(params.models.voiceLocalTtsModel);
   }
   return Array.from(ids);
@@ -260,7 +260,7 @@ export async function initializeLocalSpeechServices(params: {
     }
   };
 
-  if (providers.voiceSttProvider === "local") {
+  if (providers.voiceStt.provider === "local") {
     if (!localConfig) {
       logger.warn(
         { configured: false },
@@ -276,7 +276,7 @@ export async function initializeLocalSpeechServices(params: {
     }
   }
 
-  if (providers.dictationSttProvider === "local") {
+  if (providers.dictationStt.provider === "local") {
     if (!localConfig) {
       logger.warn(
         { configured: false },
@@ -299,7 +299,7 @@ export async function initializeLocalSpeechServices(params: {
     }
   }
 
-  if (providers.voiceTtsProvider === "local") {
+  if (providers.voiceTts.provider === "local") {
     if (!localConfig) {
       logger.warn(
         { configured: false },
@@ -323,8 +323,8 @@ export async function initializeLocalSpeechServices(params: {
             {
               preset: localModels.voiceLocalTtsModel,
               modelDir,
-              speakerId: speechConfig?.voiceLocalTtsSpeakerId,
+              speakerId: speechConfig?.localModels.voiceTtsSpeakerId,
-              speed: speechConfig?.voiceLocalTtsSpeed,
+              speed: speechConfig?.localModels.voiceTtsSpeed,
             },
             logger
           );

packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts

@@ -115,23 +115,27 @@ describe("speech models (download E2E)", () => {
         paseoHomeRoot,
         dictationFinalTimeoutMs: 8000,
         speech: {
-          dictationSttProvider: "local",
+          providers: {
-          voiceSttProvider: "local",
+            dictationStt: { provider: "local", explicit: true },
-          voiceTtsProvider: "local",
+            voiceStt: { provider: "local", explicit: true },
+            voiceTts: { provider: "local", explicit: true },
+          },
           local: {
             modelsDir,
             autoDownload: false,
           },
-          dictationLocalSttModel:
+          localModels: {
-            set === "parakeet-pocket"
+            dictationStt:
-              ? "parakeet-tdt-0.6b-v3-int8"
+              set === "parakeet-pocket"
-              : "zipformer-bilingual-zh-en-2023-02-20",
+                ? "parakeet-tdt-0.6b-v3-int8"
-          voiceLocalSttModel:
+                : "zipformer-bilingual-zh-en-2023-02-20",
-            set === "parakeet-pocket"
+            voiceStt:
-              ? "parakeet-tdt-0.6b-v3-int8"
+              set === "parakeet-pocket"
-              : "zipformer-bilingual-zh-en-2023-02-20",
+                ? "parakeet-tdt-0.6b-v3-int8"
-          voiceLocalTtsModel:
+                : "zipformer-bilingual-zh-en-2023-02-20",
-            set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
+            voiceTts:
+              set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
+          },
         },
       });
 

packages/server/src/server/speech/providers/openai/config.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, test } from "vitest";
+
+import { PersistedConfigSchema } from "../../../persisted-config.js";
+import { resolveOpenAiSpeechConfig } from "./config.js";
+
+describe("resolveOpenAiSpeechConfig", () => {
+  test("treats empty OPENAI_API_KEY as unset", () => {
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {
+      OPENAI_API_KEY: "",
+    } as NodeJS.ProcessEnv;
+
+    const resolved = resolveOpenAiSpeechConfig({
+      env,
+      persisted,
+      providers: {
+        dictationStt: { provider: "local", explicit: false },
+        voiceStt: { provider: "local", explicit: false },
+        voiceTts: { provider: "local", explicit: false },
+      },
+    });
+
+    expect(resolved).toBeUndefined();
+  });
+
+  test("uses trimmed OPENAI_API_KEY when configured", () => {
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {
+      OPENAI_API_KEY: "  sk-test  ",
+    } as NodeJS.ProcessEnv;
+
+    const resolved = resolveOpenAiSpeechConfig({
+      env,
+      persisted,
+      providers: {
+        dictationStt: { provider: "openai", explicit: true },
+        voiceStt: { provider: "openai", explicit: true },
+        voiceTts: { provider: "openai", explicit: true },
+      },
+    });
+
+    expect(resolved?.apiKey).toBe("sk-test");
+    expect(resolved?.stt?.apiKey).toBe("sk-test");
+    expect(resolved?.tts?.apiKey).toBe("sk-test");
+  });
+});

packages/server/src/server/speech/providers/openai/config.ts

@@ -35,7 +35,11 @@ const OptionalFiniteNumberSchema = NumberLikeSchema
   .pipe(z.coerce.number().finite())
   .optional();
 
-const OptionalTrimmedStringSchema = z.string().trim().min(1).optional();
+const OptionalTrimmedStringSchema = z
+  .string()
+  .trim()
+  .optional()
+  .transform((value) => (value && value.length > 0 ? value : undefined));
 
 const OpenAiSpeechResolutionSchema = z.object({
   apiKey: OptionalTrimmedStringSchema,
@@ -70,27 +74,27 @@ export function resolveOpenAiSpeechConfig(params: {
       params.persisted.features?.dictation?.stt?.confidenceThreshold,
     sttModel:
       params.env.STT_MODEL ??
-      (params.providers.voiceSttProvider === "openai"
+      (params.providers.voiceStt.provider === "openai"
         ? params.persisted.features?.voiceMode?.stt?.model
         : undefined) ??
-      (params.providers.dictationSttProvider === "openai"
+      (params.providers.dictationStt.provider === "openai"
         ? params.persisted.features?.dictation?.stt?.model
         : undefined),
     ttsVoice:
       params.env.TTS_VOICE ??
-      (params.providers.voiceTtsProvider === "openai"
+      (params.providers.voiceTts.provider === "openai"
         ? params.persisted.features?.voiceMode?.tts?.voice
         : undefined) ??
       "alloy",
     ttsModel:
       params.env.TTS_MODEL ??
-      (params.providers.voiceTtsProvider === "openai"
+      (params.providers.voiceTts.provider === "openai"
         ? params.persisted.features?.voiceMode?.tts?.model
         : undefined) ??
       DEFAULT_OPENAI_TTS_MODEL,
     realtimeTranscriptionModel:
       params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ??
-      (params.providers.dictationSttProvider === "openai"
+      (params.providers.dictationStt.provider === "openai"
         ? params.persisted.features?.dictation?.stt?.model
         : undefined) ??
       DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,

packages/server/src/server/speech/providers/openai/runtime.ts

@@ -60,13 +60,16 @@ export function validateOpenAiCredentialRequirements(params: {
   const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
 
   const missingOpenAiCredentialsFor: string[] = [];
-  if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) {
+  if (providers.voiceStt.provider === "openai" && !openAiCredentials.openaiSttApiKey) {
     missingOpenAiCredentialsFor.push("voice.stt");
   }
-  if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) {
+  if (providers.voiceTts.provider === "openai" && !openAiCredentials.openaiTtsApiKey) {
     missingOpenAiCredentialsFor.push("voice.tts");
   }
-  if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) {
+  if (
+    providers.dictationStt.provider === "openai" &&
+    !openAiCredentials.openaiDictationApiKey
+  ) {
     missingOpenAiCredentialsFor.push("dictation.stt");
   }
 
@@ -74,9 +77,9 @@ export function validateOpenAiCredentialRequirements(params: {
     logger.error(
       {
         requestedProviders: {
-          dictationStt: providers.dictationSttProvider,
+          dictationStt: providers.dictationStt.provider,
-          voiceStt: providers.voiceSttProvider,
+          voiceStt: providers.voiceStt.provider,
-          voiceTts: providers.voiceTtsProvider,
+          voiceTts: providers.voiceTts.provider,
         },
         missingOpenAiCredentialsFor,
       },
@@ -101,9 +104,10 @@ export function initializeOpenAiSpeechServices(params: {
   let ttsService = existing.ttsService;
   let dictationSttService = existing.dictationSttService;
 
-  const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai";
+  const needsOpenAiStt = !sttService && providers.voiceStt.provider === "openai";
-  const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai";
+  const needsOpenAiTts = !ttsService && providers.voiceTts.provider === "openai";
-  const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai";
+  const needsOpenAiDictation =
+    !dictationSttService && providers.dictationStt.provider === "openai";
 
   if (
     (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&

packages/server/src/server/speech/speech-config-resolver.test.ts

@@ -18,16 +18,25 @@ describe("resolveSpeechConfig", () => {
     });
 
     expect(result.openai).toBeUndefined();
-    expect(result.speech.dictationSttProvider).toBe("local");
+    expect(result.speech.providers.dictationStt).toEqual({
-    expect(result.speech.voiceSttProvider).toBe("local");
+      provider: "local",
-    expect(result.speech.voiceTtsProvider).toBe("local");
+      explicit: false,
+    });
+    expect(result.speech.providers.voiceStt).toEqual({
+      provider: "local",
+      explicit: false,
+    });
+    expect(result.speech.providers.voiceTts).toEqual({
+      provider: "local",
+      explicit: false,
+    });
     expect(result.speech.local).toEqual({
       modelsDir: path.join(paseoHome, "models", "local-speech"),
       autoDownload: true,
     });
-    expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
-    expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
-    expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8");
+    expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8");
   });
 
   test("resolves feature-scoped local model env vars", () => {
@@ -65,11 +74,23 @@ describe("resolveSpeechConfig", () => {
       modelsDir: "/tmp/models",
       autoDownload: false,
     });
-    expect(result.speech.dictationLocalSttModel).toBe("zipformer-bilingual-zh-en-2023-02-20");
+    expect(result.speech.providers.dictationStt).toEqual({
-    expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+      provider: "local",
-    expect(result.speech.voiceLocalTtsModel).toBe("kitten-nano-en-v0_1-fp16");
+      explicit: true,
-    expect(result.speech.voiceLocalTtsSpeakerId).toBe(5);
+    });
-    expect(result.speech.voiceLocalTtsSpeed).toBe(1.35);
+    expect(result.speech.providers.voiceStt).toEqual({
+      provider: "openai",
+      explicit: true,
+    });
+    expect(result.speech.providers.voiceTts).toEqual({
+      provider: "local",
+      explicit: true,
+    });
+    expect(result.speech.localModels.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20");
+    expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.voiceTts).toBe("kitten-nano-en-v0_1-fp16");
+    expect(result.speech.localModels.voiceTtsSpeakerId).toBe(5);
+    expect(result.speech.localModels.voiceTtsSpeed).toBe(1.35);
     expect(result.openai?.apiKey).toBe("env-key");
     expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe");
   });
@@ -87,8 +108,8 @@ describe("resolveSpeechConfig", () => {
       persisted,
     });
 
-    expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
-    expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
-    expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8");
+    expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8");
   });
 });

packages/server/src/server/speech/speech-config-resolver.ts

@@ -6,6 +6,7 @@ import { resolveLocalSpeechConfig } from "./providers/local/config.js";
 import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js";
 import {
   SpeechProviderIdSchema,
+  type RequestedSpeechProvider,
   type RequestedSpeechProviders,
 } from "./speech-types.js";
 
@@ -17,29 +18,53 @@ const OptionalSpeechProviderSchema = z
   .optional();
 
 const RequestedSpeechProvidersSchema = z.object({
-  dictationSttProvider: OptionalSpeechProviderSchema.default("local"),
+  dictationStt: OptionalSpeechProviderSchema.default("local"),
-  voiceSttProvider: OptionalSpeechProviderSchema.default("local"),
+  voiceStt: OptionalSpeechProviderSchema.default("local"),
-  voiceTtsProvider: OptionalSpeechProviderSchema.default("local"),
+  voiceTts: OptionalSpeechProviderSchema.default("local"),
 });
 
 function resolveRequestedSpeechProviders(params: {
   env: NodeJS.ProcessEnv;
   persisted: PersistedConfig;
 }): RequestedSpeechProviders {
-  return RequestedSpeechProvidersSchema.parse({
+  const resolveFeatureProvider = (
-    dictationSttProvider:
+    configuredValue: string | undefined,
-      params.env.PASEO_DICTATION_STT_PROVIDER ??
+    parsedValue: z.infer<typeof SpeechProviderIdSchema>
-      params.persisted.features?.dictation?.stt?.provider ??
+  ): RequestedSpeechProvider => ({
-      "local",
+    provider: parsedValue,
-    voiceSttProvider:
+    explicit: configuredValue !== undefined,
-      params.env.PASEO_VOICE_STT_PROVIDER ??
-      params.persisted.features?.voiceMode?.stt?.provider ??
-      "local",
-    voiceTtsProvider:
-      params.env.PASEO_VOICE_TTS_PROVIDER ??
-      params.persisted.features?.voiceMode?.tts?.provider ??
-      "local",
   });
+
+  const dictationSttProviderFromConfig =
+    params.env.PASEO_DICTATION_STT_PROVIDER ??
+    params.persisted.features?.dictation?.stt?.provider;
+  const voiceSttProviderFromConfig =
+    params.env.PASEO_VOICE_STT_PROVIDER ??
+    params.persisted.features?.voiceMode?.stt?.provider;
+  const voiceTtsProviderFromConfig =
+    params.env.PASEO_VOICE_TTS_PROVIDER ??
+    params.persisted.features?.voiceMode?.tts?.provider;
+
+  const parsed = RequestedSpeechProvidersSchema.parse({
+    dictationStt: dictationSttProviderFromConfig ?? "local",
+    voiceStt: voiceSttProviderFromConfig ?? "local",
+    voiceTts: voiceTtsProviderFromConfig ?? "local",
+  });
+
+  return {
+    dictationStt: resolveFeatureProvider(
+      dictationSttProviderFromConfig,
+      parsed.dictationStt
+    ),
+    voiceStt: resolveFeatureProvider(
+      voiceSttProviderFromConfig,
+      parsed.voiceStt
+    ),
+    voiceTts: resolveFeatureProvider(
+      voiceTtsProviderFromConfig,
+      parsed.voiceTts
+    ),
+  };
 }
 
 export function resolveSpeechConfig(params: {
@@ -71,21 +96,21 @@ export function resolveSpeechConfig(params: {
   return {
     openai,
     speech: {
-      dictationSttProvider: providers.dictationSttProvider,
+      providers,
-      voiceSttProvider: providers.voiceSttProvider,
-      voiceTtsProvider: providers.voiceTtsProvider,
       ...(local.local
         ? { local: local.local }
         : {}),
-      dictationLocalSttModel: local.dictationLocalSttModel,
+      localModels: {
-      voiceLocalSttModel: local.voiceLocalSttModel,
+        dictationStt: local.dictationLocalSttModel,
-      voiceLocalTtsModel: local.voiceLocalTtsModel,
+        voiceStt: local.voiceLocalSttModel,
-      ...(local.voiceLocalTtsSpeakerId !== undefined
+        voiceTts: local.voiceLocalTtsModel,
-        ? { voiceLocalTtsSpeakerId: local.voiceLocalTtsSpeakerId }
+        ...(local.voiceLocalTtsSpeakerId !== undefined
-        : {}),
+          ? { voiceTtsSpeakerId: local.voiceLocalTtsSpeakerId }
-      ...(local.voiceLocalTtsSpeed !== undefined
+          : {}),
-        ? { voiceLocalTtsSpeed: local.voiceLocalTtsSpeed }
+        ...(local.voiceLocalTtsSpeed !== undefined
-        : {}),
+          ? { voiceTtsSpeed: local.voiceLocalTtsSpeed }
+          : {}),
+      },
     },
   };
 }

packages/server/src/server/speech/speech-runtime.ts

@@ -17,10 +17,15 @@ import type { RequestedSpeechProviders } from "./speech-types.js";
 function resolveRequestedSpeechProviders(
   speechConfig: PaseoSpeechConfig | null
 ): RequestedSpeechProviders {
+  const fromConfig = speechConfig?.providers;
+  if (fromConfig) {
+    return fromConfig;
+  }
+
   return {
-    dictationSttProvider: speechConfig?.dictationSttProvider ?? "local",
+    dictationStt: { provider: "local", explicit: false },
-    voiceSttProvider: speechConfig?.voiceSttProvider ?? "local",
+    voiceStt: { provider: "local", explicit: false },
-    voiceTtsProvider: speechConfig?.voiceTtsProvider ?? "local",
+    voiceTts: { provider: "local", explicit: false },
   };
 }
 
@@ -54,9 +59,9 @@ export async function initializeSpeechRuntime(params: {
   logger.info(
     {
       requestedProviders: {
-        dictationStt: providers.dictationSttProvider,
+        dictationStt: providers.dictationStt.provider,
-        voiceStt: providers.voiceSttProvider,
+        voiceStt: providers.voiceStt.provider,
-        voiceTts: providers.voiceTtsProvider,
+        voiceTts: providers.voiceTts.provider,
       },
       availability: {
         openai: getOpenAiSpeechAvailability(openaiConfig),
@@ -98,30 +103,66 @@ export async function initializeSpeechRuntime(params: {
     !openAiSpeech.sttService ? "voice.stt" : null,
     !openAiSpeech.ttsService ? "voice.tts" : null,
   ].filter((feature): feature is string => feature !== null);
+  const explicitlyConfiguredUnavailableFeatures = unavailableFeatures.filter((feature) => {
+    if (feature === "dictation.stt") {
+      return providers.dictationStt.explicit;
+    }
+    if (feature === "voice.stt") {
+      return providers.voiceStt.explicit;
+    }
+    return providers.voiceTts.explicit;
+  });
 
-  if (unavailableFeatures.length > 0) {
+  if (explicitlyConfiguredUnavailableFeatures.length > 0) {
     logger.error(
       {
         requestedProviders: {
-          dictationStt: providers.dictationSttProvider,
+          dictationStt: providers.dictationStt.provider,
-          voiceStt: providers.voiceSttProvider,
+          voiceStt: providers.voiceStt.provider,
-          voiceTts: providers.voiceTtsProvider,
+          voiceTts: providers.voiceTts.provider,
+        },
+        explicitProviders: {
+          dictationStt: providers.dictationStt.explicit,
+          voiceStt: providers.voiceStt.explicit,
+          voiceTts: providers.voiceTts.explicit,
+        },
+        effectiveProviders,
+        unavailableFeatures: explicitlyConfiguredUnavailableFeatures,
+      },
+      "Speech provider reconciliation failed: configured features are unavailable"
+    );
+    throw new Error(
+      `Configured speech features unavailable: ${explicitlyConfiguredUnavailableFeatures.join(", ")}`
+    );
+  }
+
+  if (unavailableFeatures.length > 0) {
+    logger.warn(
+      {
+        requestedProviders: {
+          dictationStt: providers.dictationStt.provider,
+          voiceStt: providers.voiceStt.provider,
+          voiceTts: providers.voiceTts.provider,
+        },
+        explicitProviders: {
+          dictationStt: providers.dictationStt.explicit,
+          voiceStt: providers.voiceStt.explicit,
+          voiceTts: providers.voiceTts.explicit,
         },
         effectiveProviders,
         unavailableFeatures,
       },
-      "Speech provider reconciliation failed: configured features are unavailable"
+      "Speech provider reconciliation completed with unavailable default features"
+    );
+  } else {
+    logger.info(
+      {
+        effectiveProviders,
+      },
+      "Speech provider reconciliation completed"
     );
-    throw new Error(`Configured speech features unavailable: ${unavailableFeatures.join(", ")}`);
   }
 
-  logger.info(
-    {
-      effectiveProviders,
-    },
-    "Speech provider reconciliation completed"
-  );
-
   return {
     sttService: openAiSpeech.sttService,
     ttsService: openAiSpeech.ttsService,

packages/server/src/server/speech/speech-types.ts

@@ -3,8 +3,14 @@ import { z } from "zod";
 export const SpeechProviderIdSchema = z.enum(["openai", "local"]);
 export type SpeechProviderId = z.infer<typeof SpeechProviderIdSchema>;
 
+export const RequestedSpeechProviderSchema = z.object({
+  provider: SpeechProviderIdSchema,
+  explicit: z.boolean(),
+});
+export type RequestedSpeechProvider = z.infer<typeof RequestedSpeechProviderSchema>;
+
 export type RequestedSpeechProviders = {
-  dictationSttProvider: SpeechProviderId;
+  dictationStt: RequestedSpeechProvider;
-  voiceSttProvider: SpeechProviderId;
+  voiceStt: RequestedSpeechProvider;
-  voiceTtsProvider: SpeechProviderId;
+  voiceTts: RequestedSpeechProvider;
 };

packages/server/src/server/voice-local-agent.e2e.test.ts

@@ -48,9 +48,16 @@ function waitForSignal<T>(
         agentClients: {},
         openai: { apiKey: openaiApiKey! },
         speech: {
-          dictationSttProvider: "openai",
+          providers: {
-          voiceSttProvider: "openai",
+            dictationStt: { provider: "openai", explicit: true },
-          voiceTtsProvider: "openai",
+            voiceStt: { provider: "openai", explicit: true },
+            voiceTts: { provider: "openai", explicit: true },
+          },
+          localModels: {
+            dictationStt: "parakeet-tdt-0.6b-v3-int8",
+            voiceStt: "parakeet-tdt-0.6b-v3-int8",
+            voiceTts: "pocket-tts-onnx-int8",
+          },
         },
         voiceLlmProvider: "codex",
         voiceLlmProviderExplicit: true,