speech: address PR review feedback and remove unsafe casts

packages/server/src/server/agent/agent-management-mcp.ts

@@ -23,8 +23,6 @@
 
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
-import { homedir } from "node:os";
-import { resolve } from "node:path";
 import { ensureValidJson } from "../json-utils.js";
 import type { Logger } from "pino";
 
@@ -51,6 +49,7 @@ import { AgentStorage } from "./agent-storage.js";
 import { createWorktree } from "../../utils/worktree.js";
 import { WaitForAgentTracker } from "./wait-for-agent-tracker.js";
 import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js";
+import { expandUserPath } from "../path-utils.js";
 
 export interface AgentManagementMcpOptions {
   agentManager: AgentManager;
@@ -77,13 +76,6 @@ const AgentStatusEnum = z.enum([
 // 50 seconds - surface friendly message before SDK tool timeout (~60s)
 const AGENT_WAIT_TIMEOUT_MS = 50000;
 
-function expandPath(path: string): string {
-  if (path.startsWith("~/") || path === "~") {
-    return resolve(homedir(), path.slice(2));
-  }
-  return resolve(path);
-}
-
 async function waitForAgentWithTimeout(
   agentManager: AgentManager,
   agentId: string,
@@ -320,7 +312,7 @@ export async function createAgentManagementMcpServer(
         title: string;
       };
 
-      let resolvedCwd = expandPath(cwd);
+      let resolvedCwd = expandUserPath(cwd);
 
       if (worktreeName) {
         if (!baseBranch) {

packages/server/src/server/agent/mcp-server.ts

@@ -1,9 +1,12 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
-import { homedir } from "node:os";
-import { resolve } from "node:path";
 import { ensureValidJson } from "../json-utils.js";
 import type { Logger } from "pino";
+import type { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js";
+import type {
+  ServerNotification,
+  ServerRequest,
+} from "@modelcontextprotocol/sdk/types.js";
 
 import type {
   AgentPromptInput,
@@ -32,6 +35,7 @@ import type {
   VoiceCallerContext,
   VoiceSpeakHandler,
 } from "../voice-types.js";
+import { expandUserPath, resolvePathFromBase } from "../path-utils.js";
 
 export interface AgentMcpServerOptions {
   agentManager: AgentManager;
@@ -115,11 +119,25 @@ const AgentStatusEnum = z.enum([
 // 50 seconds - surface friendly message before SDK tool timeout (~60s)
 const AGENT_WAIT_TIMEOUT_MS = 50000;
 
-function expandPath(path: string): string {
+type McpToolContext = RequestHandlerExtra<ServerRequest, ServerNotification>;
-  if (path.startsWith("~/") || path === "~") {
+
-    return resolve(homedir(), path.slice(2));
+function resolveChildAgentCwd(params: {
+  parentCwd: string;
+  requestedCwd?: string;
+  lockedCwd?: string;
+  allowCustomCwd: boolean;
+}): string {
+  const lockedCwd = params.lockedCwd?.trim();
+  if (lockedCwd) {
+    return expandUserPath(lockedCwd);
   }
-  return resolve(path);
+
+  const requestedCwd = params.requestedCwd?.trim();
+  if (!requestedCwd || !params.allowCustomCwd) {
+    return params.parentCwd;
+  }
+
+  return resolvePathFromBase(params.parentCwd, requestedCwd);
 }
 
 /**
@@ -369,6 +387,11 @@ export async function createAgentMcpServer(
   const createAgentInputSchema = callerAgentId
     ? agentToAgentInputSchema
     : topLevelInputSchema;
+  const agentToAgentCreateAgentArgsSchema = z.object(agentToAgentInputSchema);
+  const topLevelCreateAgentArgsSchema = z.object({
+    ...topLevelInputSchema,
+    initialMode: topLevelInputSchema.initialMode.optional(),
+  });
 
   if (options.enableVoiceTools || callerContext?.enableVoiceTools) {
     server.registerTool(
@@ -388,7 +411,7 @@ export async function createAgentMcpServer(
           ok: z.boolean(),
         },
       },
-      async (args, context) => {
+      async (args, context?: McpToolContext) => {
         if (!callerAgentId) {
           throw new Error("speak is only available to agent-scoped MCP sessions");
         }
@@ -399,7 +422,7 @@ export async function createAgentMcpServer(
         await handler({
           text: args.text,
           callerAgentId,
-          signal: (context as { signal?: AbortSignal } | undefined)?.signal,
+          signal: context?.signal,
         });
         return {
           content: [],
@@ -434,44 +457,31 @@ export async function createAgentMcpServer(
       },
     },
     async (args: unknown) => {
-      const {
+      let provider: AgentProvider;
-        agentType,
+      let initialPrompt: string;
-        initialPrompt,
+      let background = false;
-        background = false,
+      let normalizedTitle: string | null;
-        title,
-      } = args as {
-        cwd?: string;
-        agentType?: AgentProvider;
-        initialPrompt: string;
-        initialMode?: string;
-        worktreeName?: string;
-        background?: boolean;
-        title: string;
-      };
 
       let resolvedCwd: string;
       let resolvedMode: string | undefined;
 
       if (callerAgentId) {
+        const callerArgs = agentToAgentCreateAgentArgsSchema.parse(args);
+        provider = callerArgs.agentType ?? "claude";
+        initialPrompt = callerArgs.initialPrompt;
+        background = callerArgs.background ?? false;
+        normalizedTitle = callerArgs.title.trim();
+
         const parentAgent = agentManager.getAgent(callerAgentId);
         if (!parentAgent) {
           throw new Error(`Parent agent ${callerAgentId} not found`);
         }
-        const callerArgs = args as unknown as { cwd?: string };
+        resolvedCwd = resolveChildAgentCwd({
-        const requestedCwd = callerArgs.cwd?.trim();
+          parentCwd: parentAgent.cwd,
-        const lockedCwd = callerContext?.lockedCwd?.trim();
+          requestedCwd: callerArgs.cwd,
-        if (lockedCwd) {
+          lockedCwd: callerContext?.lockedCwd,
-          resolvedCwd = expandPath(lockedCwd);
+          allowCustomCwd: callerContext?.allowCustomCwd ?? true,
-        } else if (requestedCwd && (callerContext?.allowCustomCwd ?? true)) {
+        });
-          resolvedCwd =
-            requestedCwd.startsWith("/") || requestedCwd.startsWith("~")
-              ? expandPath(requestedCwd)
-              : resolve(parentAgent.cwd, requestedCwd);
-        } else {
-          resolvedCwd = parentAgent.cwd;
-        }
-
-        const provider: AgentProvider = agentType ?? "claude";
         const parentMode = parentAgent.currentModeId;
         if (parentMode) {
           resolvedMode = mapModeAcrossProviders(
@@ -481,12 +491,11 @@ export async function createAgentMcpServer(
           );
         }
       } else {
-        const topLevelArgs = args as unknown as {
+        const topLevelArgs = topLevelCreateAgentArgsSchema.parse(args);
-          cwd: string;
+        provider = topLevelArgs.agentType ?? "claude";
-          initialMode: string;
+        initialPrompt = topLevelArgs.initialPrompt;
-          worktreeName?: string;
+        background = topLevelArgs.background ?? false;
-          baseBranch?: string;
+        normalizedTitle = topLevelArgs.title.trim();
-        };
         const {
           cwd,
           initialMode,
@@ -494,7 +503,7 @@ export async function createAgentMcpServer(
           baseBranch,
         } = topLevelArgs;
 
-        resolvedCwd = expandPath(cwd);
+        resolvedCwd = expandUserPath(cwd);
 
         if (worktreeName) {
           if (!baseBranch) {
@@ -513,8 +522,6 @@ export async function createAgentMcpServer(
         resolvedMode = initialMode;
       }
 
-      const provider: AgentProvider = agentType ?? "claude";
-      const normalizedTitle = title?.trim() ?? null;
       const childAgentDefaultLabels =
         callerAgentId && callerContext?.childAgentDefaultLabels
           ? callerContext.childAgentDefaultLabels

packages/server/src/server/path-utils.ts

@@ -0,0 +1,22 @@
+import { homedir } from "node:os";
+import { isAbsolute, resolve } from "node:path";
+
+function hasHomePrefix(value: string): boolean {
+  return value === "~" || value.startsWith("~/");
+}
+
+export function expandUserPath(value: string): string {
+  const trimmed = value.trim();
+  if (hasHomePrefix(trimmed)) {
+    return resolve(homedir(), trimmed.slice(2));
+  }
+  return resolve(trimmed);
+}
+
+export function resolvePathFromBase(baseCwd: string, requestedPath: string): string {
+  const trimmed = requestedPath.trim();
+  if (hasHomePrefix(trimmed) || isAbsolute(trimmed)) {
+    return expandUserPath(trimmed);
+  }
+  return resolve(baseCwd, trimmed);
+}

packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts

@@ -2,57 +2,75 @@ import { z } from "zod";
 
 export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts";
 
-export type SherpaOnnxModelId =
+const MODEL_IDS = {
-  | "zipformer-bilingual-zh-en-2023-02-20"
+  ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20: "zipformer-bilingual-zh-en-2023-02-20",
-  | "paraformer-bilingual-zh-en"
+  PARAFORMER_BILINGUAL_ZH_EN: "paraformer-bilingual-zh-en",
-  | "parakeet-tdt-0.6b-v3-int8"
+  PARAKEET_TDT_0_6B_V3_INT8: "parakeet-tdt-0.6b-v3-int8",
-  | "kitten-nano-en-v0_1-fp16"
+  KITTEN_NANO_EN_V0_1_FP16: "kitten-nano-en-v0_1-fp16",
-  | "kokoro-en-v0_19"
+  KOKORO_EN_V0_19: "kokoro-en-v0_19",
-  | "pocket-tts-onnx-int8";
+  POCKET_TTS_ONNX_INT8: "pocket-tts-onnx-int8",
+} as const;
+
+export type SherpaOnnxModelId = (typeof MODEL_IDS)[keyof typeof MODEL_IDS];
 
 export const LOCAL_STT_MODEL_IDS = [
-  "zipformer-bilingual-zh-en-2023-02-20",
+  MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20,
-  "paraformer-bilingual-zh-en",
+  MODEL_IDS.PARAFORMER_BILINGUAL_ZH_EN,
-  "parakeet-tdt-0.6b-v3-int8",
+  MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8,
 ] as const;
 export type LocalSttModelId = (typeof LOCAL_STT_MODEL_IDS)[number];
 
 export const LOCAL_TTS_MODEL_IDS = [
-  "kitten-nano-en-v0_1-fp16",
+  MODEL_IDS.KITTEN_NANO_EN_V0_1_FP16,
-  "kokoro-en-v0_19",
+  MODEL_IDS.KOKORO_EN_V0_19,
-  "pocket-tts-onnx-int8",
+  MODEL_IDS.POCKET_TTS_ONNX_INT8,
 ] as const;
 export type LocalTtsModelId = (typeof LOCAL_TTS_MODEL_IDS)[number];
 
-const STT_MODEL_ALIASES: Record<string, (typeof LOCAL_STT_MODEL_IDS)[number]> = {
+export const DEFAULT_LOCAL_STT_MODEL: LocalSttModelId = MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8;
-  zipformer: "zipformer-bilingual-zh-en-2023-02-20",
+export const DEFAULT_LOCAL_TTS_MODEL: LocalTtsModelId = MODEL_IDS.POCKET_TTS_ONNX_INT8;
-  "zipformer-bilingual": "zipformer-bilingual-zh-en-2023-02-20",
+
-  paraformer: "paraformer-bilingual-zh-en",
+const STT_MODEL_ALIASES: Record<string, LocalSttModelId> = {
-  parakeet: "parakeet-tdt-0.6b-v3-int8",
+  zipformer: MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20,
-  "parakeet-v3": "parakeet-tdt-0.6b-v3-int8",
+  "zipformer-bilingual": MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20,
-  "parakeet-tdt": "parakeet-tdt-0.6b-v3-int8",
+  paraformer: MODEL_IDS.PARAFORMER_BILINGUAL_ZH_EN,
+  parakeet: MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8,
+  "parakeet-v3": MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8,
+  "parakeet-tdt": MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8,
 };
 
-const TTS_MODEL_ALIASES: Record<string, (typeof LOCAL_TTS_MODEL_IDS)[number]> = {
+const TTS_MODEL_ALIASES: Record<string, LocalTtsModelId> = {
-  pocket: "pocket-tts-onnx-int8",
+  pocket: MODEL_IDS.POCKET_TTS_ONNX_INT8,
-  "pocket-tts": "pocket-tts-onnx-int8",
+  "pocket-tts": MODEL_IDS.POCKET_TTS_ONNX_INT8,
-  kitten: "kitten-nano-en-v0_1-fp16",
+  kitten: MODEL_IDS.KITTEN_NANO_EN_V0_1_FP16,
-  kokoro: "kokoro-en-v0_19",
+  kokoro: MODEL_IDS.KOKORO_EN_V0_19,
 };
 
-export const LocalSttModelIdSchema = z.preprocess((value) => {
+function createAliasedModelIdSchema<T extends readonly [string, ...string[]]>(
-  if (typeof value !== "string") return value;
+  values: T,
-  const normalized = value.trim().toLowerCase();
+  aliases: Record<string, T[number]>
-  if (!normalized) return value;
+) {
-  return STT_MODEL_ALIASES[normalized] ?? normalized;
+  return z.preprocess((value) => {
-}, z.enum(LOCAL_STT_MODEL_IDS));
+    if (typeof value !== "string") {
+      return value;
+    }
+    const normalized = value.trim().toLowerCase();
+    if (!normalized) {
+      return value;
+    }
+    return aliases[normalized] ?? normalized;
+  }, z.enum(values));
+}
 
-export const LocalTtsModelIdSchema = z.preprocess((value) => {
+export const LocalSttModelIdSchema = createAliasedModelIdSchema(
-  if (typeof value !== "string") return value;
+  LOCAL_STT_MODEL_IDS,
-  const normalized = value.trim().toLowerCase();
+  STT_MODEL_ALIASES
-  if (!normalized) return value;
+);
-  return TTS_MODEL_ALIASES[normalized] ?? normalized;
+
-}, z.enum(LOCAL_TTS_MODEL_IDS));
+export const LocalTtsModelIdSchema = createAliasedModelIdSchema(
+  LOCAL_TTS_MODEL_IDS,
+  TTS_MODEL_ALIASES
+);
 
 export type SherpaOnnxModelSpec = {
   id: SherpaOnnxModelId;
@@ -65,8 +83,8 @@ export type SherpaOnnxModelSpec = {
 };
 
 export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxModelSpec> = {
-  "zipformer-bilingual-zh-en-2023-02-20": {
+  [MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20]: {
-    id: "zipformer-bilingual-zh-en-2023-02-20",
+    id: MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20,
     kind: "stt-online",
     archiveUrl:
       "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2",
@@ -79,8 +97,8 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxMode
     ],
     description: "Streaming Zipformer transducer (fast, good accuracy).",
   },
-  "paraformer-bilingual-zh-en": {
+  [MODEL_IDS.PARAFORMER_BILINGUAL_ZH_EN]: {
-    id: "paraformer-bilingual-zh-en",
+    id: MODEL_IDS.PARAFORMER_BILINGUAL_ZH_EN,
     kind: "stt-online",
     archiveUrl:
       "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2",
@@ -88,8 +106,8 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxMode
     requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"],
     description: "Streaming Paraformer (often strong accuracy; heavier).",
   },
-  "parakeet-tdt-0.6b-v3-int8": {
+  [MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8]: {
-    id: "parakeet-tdt-0.6b-v3-int8",
+    id: MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8,
     kind: "stt-offline",
     archiveUrl:
       "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
@@ -97,8 +115,8 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxMode
     requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
     description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).",
   },
-  "kitten-nano-en-v0_1-fp16": {
+  [MODEL_IDS.KITTEN_NANO_EN_V0_1_FP16]: {
-    id: "kitten-nano-en-v0_1-fp16",
+    id: MODEL_IDS.KITTEN_NANO_EN_V0_1_FP16,
     kind: "tts",
     archiveUrl:
       "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2",
@@ -106,16 +124,16 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxMode
     requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
     description: "KittenTTS (small, fast English TTS).",
   },
-  "kokoro-en-v0_19": {
+  [MODEL_IDS.KOKORO_EN_V0_19]: {
-    id: "kokoro-en-v0_19",
+    id: MODEL_IDS.KOKORO_EN_V0_19,
     kind: "tts",
     archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2",
     extractedDir: "kokoro-en-v0_19",
     requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
     description: "Kokoro TTS (higher quality; larger).",
   },
-  "pocket-tts-onnx-int8": {
+  [MODEL_IDS.POCKET_TTS_ONNX_INT8]: {
-    id: "pocket-tts-onnx-int8",
+    id: MODEL_IDS.POCKET_TTS_ONNX_INT8,
     kind: "tts",
     extractedDir: "pocket-tts-onnx-int8",
     downloadFiles: [

packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts

@@ -1,6 +1,4 @@
 import { createRequire } from "node:module";
-import os from "node:os";
-import path from "node:path";
 
 export type SherpaOnnxModule = {
   createOnlineRecognizer: (config: any) => any;
@@ -10,39 +8,12 @@ export type SherpaOnnxModule = {
 
 let cached: SherpaOnnxModule | null = null;
 
-function ensureSherpaNativeLibraryPath(requireFn: NodeRequire): void {
-  const platform = os.platform();
-  if (platform !== "darwin" && platform !== "linux") {
-    return;
-  }
-
-  const platformArch = `${platform}-${os.arch()}`;
-  const packageName = `sherpa-onnx-${platformArch}`;
-
-  let nativeDir: string;
-  try {
-    const binaryPath = requireFn.resolve(`${packageName}/sherpa-onnx.node`);
-    nativeDir = path.dirname(binaryPath);
-  } catch {
-    return;
-  }
-
-  const envKey = platform === "darwin" ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH";
-  const current = process.env[envKey]?.trim() ?? "";
-  const entries = current.length > 0 ? current.split(":").filter(Boolean) : [];
-  if (entries.includes(nativeDir)) {
-    return;
-  }
-
-  process.env[envKey] = entries.length > 0 ? `${nativeDir}:${entries.join(":")}` : nativeDir;
-}
-
 export function loadSherpaOnnx(): SherpaOnnxModule {
   if (cached) {
     return cached;
   }
+
   const require = createRequire(import.meta.url);
-  ensureSherpaNativeLibraryPath(require);
   cached = require("sherpa-onnx") as SherpaOnnxModule;
   return cached;
 }

packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts

@@ -14,13 +14,16 @@ function platformArch(): string {
   return `${platform}-${process.arch}`;
 }
 
-function prependLibraryPath(envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH", dir: string): void {
+function prependLibraryPath(
+  envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH" | "PATH",
+  dir: string
+): void {
   const current = process.env[envKey] ?? "";
-  const parts = current.split(":").filter(Boolean);
+  const parts = current.split(path.delimiter).filter(Boolean);
   if (parts.includes(dir)) {
     return;
   }
-  process.env[envKey] = [dir, ...parts].join(":");
+  process.env[envKey] = [dir, ...parts].join(path.delimiter);
 }
 
 export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
@@ -31,8 +34,7 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
   const require = createRequire(import.meta.url);
 
   // sherpa-onnx-node depends on a platform-specific package (e.g. sherpa-onnx-darwin-arm64)
-  // that contains the native addon + its shared libraries. On macOS/Linux we need to ensure
+  // that contains the native addon and shared libraries. Ensure the OS loader path includes it.
-  // the appropriate library path env var includes that directory before requiring the addon.
   const arch = platformArch();
   const pkgName = `sherpa-onnx-${arch}`;
 
@@ -43,6 +45,8 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
       prependLibraryPath("DYLD_LIBRARY_PATH", pkgDir);
     } else if (process.platform === "linux") {
       prependLibraryPath("LD_LIBRARY_PATH", pkgDir);
+    } else if (process.platform === "win32") {
+      prependLibraryPath("PATH", pkgDir);
     }
   } catch {
     // Best effort - if the platform package isn't present, require() below will throw a useful error.

packages/server/src/server/speech/speech-config-resolver.ts

@@ -1,104 +1,172 @@
 import path from "node:path";
 
-import type { STTConfig } from "./providers/openai/stt.js";
+import { z } from "zod";
-import type { TTSConfig } from "./providers/openai/tts.js";
+
 import type { PersistedConfig } from "../persisted-config.js";
 import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
-import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/sherpa/model-catalog.js";
+import {
+  LocalSttModelIdSchema,
+  LocalTtsModelIdSchema,
+} from "./providers/local/sherpa/model-catalog.js";
 import {
   DEFAULT_LOCAL_STT_MODEL,
   DEFAULT_LOCAL_TTS_MODEL,
   DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
-  type SpeechProviderId,
+  DEFAULT_OPENAI_TTS_MODEL,
-} from "./speech-types.js";
+} from "./speech-defaults.js";
+import { SpeechProviderIdSchema } from "./speech-types.js";
+
 const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
-const DEFAULT_OPENAI_TTS_MODEL: TTSConfig["model"] = "tts-1";
 
-function parseSpeechProviderId(value: unknown): SpeechProviderId | null {
+const OpenAiTtsVoiceSchema = z.enum([
-  if (typeof value !== "string") return null;
+  "alloy",
+  "echo",
+  "fable",
+  "onyx",
+  "nova",
+  "shimmer",
+]);
+
+const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
+
+const OptionalSpeechProviderSchema = z.preprocess((value) => {
+  if (typeof value !== "string") {
+    return value;
+  }
   const normalized = value.trim().toLowerCase();
-  if (!normalized) return null;
+  return normalized.length > 0 ? normalized : undefined;
-  if (normalized === "openai") return "openai";
+}, SpeechProviderIdSchema.optional());
-  if (normalized === "local") return "local";
-  return null;
-}
 
-function parseBooleanFlag(value: string | undefined): boolean | null {
+const OptionalBooleanFlagSchema = z.preprocess((value) => {
-  if (value === undefined) return null;
+  if (typeof value === "boolean") {
+    return value;
+  }
+  if (typeof value !== "string") {
+    return value;
+  }
   const normalized = value.trim().toLowerCase();
-  if (normalized === "1" || normalized === "true" || normalized === "yes") return true;
+  if (normalized === "1" || normalized === "true" || normalized === "yes") {
-  if (normalized === "0" || normalized === "false" || normalized === "no") return false;
+    return true;
-  return null;
+  }
-}
+  if (normalized === "0" || normalized === "false" || normalized === "no") {
+    return false;
+  }
+  return undefined;
+}, z.boolean().optional());
 
-function parseNumberOrUndefined(value: string | undefined): number | undefined {
+const OptionalFiniteNumberSchema = z.preprocess((value) => {
-  if (value === undefined) return undefined;
+  if (typeof value === "number") {
+    return Number.isFinite(value) ? value : undefined;
+  }
+  if (typeof value !== "string") {
+    return value;
+  }
   const parsed = Number.parseFloat(value);
   return Number.isFinite(parsed) ? parsed : undefined;
-}
+}, z.number().optional());
 
-function parseIntOrUndefined(value: string | undefined): number | undefined {
+const OptionalIntegerSchema = z.preprocess((value) => {
-  if (value === undefined) return undefined;
+  if (typeof value === "number") {
+    return Number.isInteger(value) ? value : undefined;
+  }
+  if (typeof value !== "string") {
+    return value;
+  }
   const parsed = Number.parseInt(value, 10);
   return Number.isFinite(parsed) ? parsed : undefined;
-}
+}, z.number().int().optional());
 
-function parseOpenAiConfig(
+const OptionalTrimmedStringSchema = z.preprocess((value) => {
-  env: NodeJS.ProcessEnv,
+  if (typeof value !== "string") {
-  persisted: PersistedConfig,
+    return value;
-  providers: {
-    dictationSttProvider: SpeechProviderId;
-    voiceSttProvider: SpeechProviderId;
-    voiceTtsProvider: SpeechProviderId;
   }
-): PaseoOpenAIConfig | undefined {
+  const trimmed = value.trim();
-  const apiKey = env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey;
+  return trimmed.length > 0 ? trimmed : undefined;
-  if (!apiKey) return undefined;
+}, z.string().optional());
 
-  const sttConfidenceThreshold = parseNumberOrUndefined(env.STT_CONFIDENCE_THRESHOLD)
+const ResolvedSpeechConfigSchema = z
-    ?? persisted.features?.dictation?.stt?.confidenceThreshold;
+  .object({
+    dictationSttProvider: OptionalSpeechProviderSchema.default("local"),
+    voiceSttProvider: OptionalSpeechProviderSchema.default("local"),
+    voiceTtsProvider: OptionalSpeechProviderSchema.default("local"),
+    localModelsDir: z.string().trim().min(1),
+    localAutoDownload: OptionalBooleanFlagSchema.default(true),
+    dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
+    voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
+    voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL),
+    voiceLocalTtsSpeakerId: OptionalIntegerSchema,
+    voiceLocalTtsSpeed: OptionalFiniteNumberSchema,
+    anyLocalRequested: z.boolean(),
+    openaiApiKey: OptionalTrimmedStringSchema,
+    openaiSttConfidenceThreshold: OptionalFiniteNumberSchema,
+    openaiSttModel: OptionalTrimmedStringSchema,
+    openaiTtsVoice: z.preprocess((value) => {
+      if (typeof value !== "string") {
+        return value;
+      }
+      const normalized = value.trim().toLowerCase();
+      return normalized.length > 0 ? normalized : undefined;
+    }, OpenAiTtsVoiceSchema.default("alloy")),
+    openaiTtsModel: z.preprocess((value) => {
+      if (typeof value !== "string") {
+        return value;
+      }
+      const normalized = value.trim().toLowerCase();
+      return normalized.length > 0 ? normalized : undefined;
+    }, OpenAiTtsModelSchema.default(DEFAULT_OPENAI_TTS_MODEL)),
+    openaiRealtimeTranscriptionModel: OptionalTrimmedStringSchema.default(
+      DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL
+    ),
+  })
+  .transform((input): { openai: PaseoOpenAIConfig | undefined; speech: PaseoSpeechConfig } => {
+    const openai = input.openaiApiKey
+      ? {
+          apiKey: input.openaiApiKey,
+          stt: {
+            apiKey: input.openaiApiKey,
+            ...(input.openaiSttConfidenceThreshold !== undefined
+              ? { confidenceThreshold: input.openaiSttConfidenceThreshold }
+              : {}),
+            ...(input.openaiSttModel
+              ? { model: input.openaiSttModel }
+              : {}),
+          },
+          tts: {
+            apiKey: input.openaiApiKey,
+            voice: input.openaiTtsVoice,
+            model: input.openaiTtsModel,
+            responseFormat: "pcm" as const,
+          },
+          realtimeTranscriptionModel: input.openaiRealtimeTranscriptionModel,
+        }
+      : undefined;
 
-  const sttModel = (
+    return {
-    env.STT_MODEL
+      openai,
-    ?? (providers.voiceSttProvider === "openai" ? persisted.features?.voiceMode?.stt?.model : undefined)
+      speech: {
-    ?? (providers.dictationSttProvider === "openai" ? persisted.features?.dictation?.stt?.model : undefined)
+        dictationSttProvider: input.dictationSttProvider,
-  ) as STTConfig["model"] | undefined;
+        voiceSttProvider: input.voiceSttProvider,
-
+        voiceTtsProvider: input.voiceTtsProvider,
-  const ttsVoice = (
+        ...(input.anyLocalRequested
-    env.TTS_VOICE
+          ? {
-    || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.voice : undefined)
+              local: {
-    || "alloy"
+                modelsDir: input.localModelsDir,
-  ) as TTSConfig["voice"];
+                autoDownload: input.localAutoDownload,
-  const ttsModelRaw =
+              },
-    env.TTS_MODEL
+            }
-    || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.model : undefined)
+          : {}),
-    || DEFAULT_OPENAI_TTS_MODEL;
+        dictationLocalSttModel: input.dictationLocalSttModel,
-  const ttsModel: TTSConfig["model"] =
+        voiceLocalSttModel: input.voiceLocalSttModel,
-    ttsModelRaw === "tts-1" || ttsModelRaw === "tts-1-hd" ? ttsModelRaw : DEFAULT_OPENAI_TTS_MODEL;
+        voiceLocalTtsModel: input.voiceLocalTtsModel,
-
+        ...(input.voiceLocalTtsSpeakerId !== undefined
-  const realtimeTranscriptionModel =
+          ? { voiceLocalTtsSpeakerId: input.voiceLocalTtsSpeakerId }
-    env.OPENAI_REALTIME_TRANSCRIPTION_MODEL
+          : {}),
-    || (providers.dictationSttProvider === "openai"
+        ...(input.voiceLocalTtsSpeed !== undefined
-      ? persisted.features?.dictation?.stt?.model
+          ? { voiceLocalTtsSpeed: input.voiceLocalTtsSpeed }
-      : undefined)
+          : {}),
-    || DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL;
+      },
-
+    };
-  return {
+  });
-    apiKey,
-    stt: {
-      apiKey,
-      ...(sttConfidenceThreshold !== undefined ? { confidenceThreshold: sttConfidenceThreshold } : {}),
-      ...(sttModel ? { model: sttModel } : {}),
-    },
-    tts: {
-      apiKey,
-      voice: ttsVoice,
-      model: ttsModel,
-      responseFormat: "pcm",
-    },
-    realtimeTranscriptionModel,
-  };
-}
 
 export function resolveSpeechConfig(params: {
   paseoHome: string;
@@ -111,18 +179,18 @@ export function resolveSpeechConfig(params: {
   const { paseoHome, env, persisted } = params;
 
   const dictationSttProvider =
-    parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER)
+    env.PASEO_DICTATION_STT_PROVIDER
-    ?? parseSpeechProviderId(persisted.features?.dictation?.stt?.provider)
+    ?? persisted.features?.dictation?.stt?.provider
     ?? "local";
 
   const voiceSttProvider =
-    parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER)
+    env.PASEO_VOICE_STT_PROVIDER
-    ?? parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider)
+    ?? persisted.features?.voiceMode?.stt?.provider
     ?? "local";
 
   const voiceTtsProvider =
-    parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER)
+    env.PASEO_VOICE_TTS_PROVIDER
-    ?? parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider)
+    ?? persisted.features?.voiceMode?.tts?.provider
     ?? "local";
 
   const anyLocalRequested =
@@ -132,65 +200,65 @@ export function resolveSpeechConfig(params: {
     env.PASEO_LOCAL_MODELS_DIR !== undefined ||
     persisted.providers?.local !== undefined;
 
-  const localModelsDir =
+  return ResolvedSpeechConfigSchema.parse({
-    env.PASEO_LOCAL_MODELS_DIR
+    dictationSttProvider,
-    ?? persisted.providers?.local?.modelsDir
+    voiceSttProvider,
-    ?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR);
+    voiceTtsProvider,
-
+    localModelsDir:
-  const localAutoDownload =
+      env.PASEO_LOCAL_MODELS_DIR
-    parseBooleanFlag(env.PASEO_LOCAL_AUTO_DOWNLOAD)
+      ?? persisted.providers?.local?.modelsDir
-    ?? persisted.providers?.local?.autoDownload
+      ?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR),
-    ?? true;
+    localAutoDownload:
-
+      env.PASEO_LOCAL_AUTO_DOWNLOAD
-  const dictationLocalSttModel = LocalSttModelIdSchema.parse(
+      ?? persisted.providers?.local?.autoDownload,
-    env.PASEO_DICTATION_LOCAL_STT_MODEL
+    dictationLocalSttModel:
-    ?? persisted.features?.dictation?.stt?.model
+      env.PASEO_DICTATION_LOCAL_STT_MODEL
-    ?? DEFAULT_LOCAL_STT_MODEL
+      ?? persisted.features?.dictation?.stt?.model
-  );
+      ?? DEFAULT_LOCAL_STT_MODEL,
-
+    voiceLocalSttModel:
-  const voiceLocalSttModel = LocalSttModelIdSchema.parse(
+      env.PASEO_VOICE_LOCAL_STT_MODEL
-    env.PASEO_VOICE_LOCAL_STT_MODEL
+      ?? persisted.features?.voiceMode?.stt?.model
-    ?? persisted.features?.voiceMode?.stt?.model
+      ?? DEFAULT_LOCAL_STT_MODEL,
-    ?? DEFAULT_LOCAL_STT_MODEL
+    voiceLocalTtsModel:
-  );
+      env.PASEO_VOICE_LOCAL_TTS_MODEL
-
+      ?? persisted.features?.voiceMode?.tts?.model
-  const voiceLocalTtsModel = LocalTtsModelIdSchema.parse(
+      ?? DEFAULT_LOCAL_TTS_MODEL,
-    env.PASEO_VOICE_LOCAL_TTS_MODEL
+    voiceLocalTtsSpeakerId:
-    ?? persisted.features?.voiceMode?.tts?.model
+      env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID
-    ?? DEFAULT_LOCAL_TTS_MODEL
+      ?? persisted.features?.voiceMode?.tts?.speakerId,
-  );
+    voiceLocalTtsSpeed:
-
+      env.PASEO_VOICE_LOCAL_TTS_SPEED
-  const voiceLocalTtsSpeakerId =
+      ?? persisted.features?.voiceMode?.tts?.speed,
-    parseIntOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID)
+    anyLocalRequested,
-    ?? persisted.features?.voiceMode?.tts?.speakerId;
+    openaiApiKey: env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey,
-
+    openaiSttConfidenceThreshold:
-  const voiceLocalTtsSpeed =
+      env.STT_CONFIDENCE_THRESHOLD
-    parseNumberOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEED)
+      ?? persisted.features?.dictation?.stt?.confidenceThreshold,
-    ?? persisted.features?.voiceMode?.tts?.speed;
+    openaiSttModel:
-
+      env.STT_MODEL
-  return {
+      ?? (voiceSttProvider === "openai"
-    openai: parseOpenAiConfig(env, persisted, {
+        ? persisted.features?.voiceMode?.stt?.model
-      dictationSttProvider,
+        : undefined)
-      voiceSttProvider,
+      ?? (dictationSttProvider === "openai"
-      voiceTtsProvider,
+        ? persisted.features?.dictation?.stt?.model
-    }),
+        : undefined),
-    speech: {
+    openaiTtsVoice:
-      dictationSttProvider,
+      env.TTS_VOICE
-      voiceSttProvider,
+      ?? (voiceTtsProvider === "openai"
-      voiceTtsProvider,
+        ? persisted.features?.voiceMode?.tts?.voice
-      ...(anyLocalRequested
+        : undefined)
-        ? {
+      ?? "alloy",
-            local: {
+    openaiTtsModel:
-              modelsDir: localModelsDir.trim(),
+      env.TTS_MODEL
-              autoDownload: localAutoDownload,
+      ?? (voiceTtsProvider === "openai"
-            },
+        ? persisted.features?.voiceMode?.tts?.model
-          }
+        : undefined)
-        : {}),
+      ?? DEFAULT_OPENAI_TTS_MODEL,
-      dictationLocalSttModel,
+    openaiRealtimeTranscriptionModel:
-      voiceLocalSttModel,
+      env.OPENAI_REALTIME_TRANSCRIPTION_MODEL
-      voiceLocalTtsModel,
+      ?? (dictationSttProvider === "openai"
-      ...(voiceLocalTtsSpeakerId !== undefined ? { voiceLocalTtsSpeakerId } : {}),
+        ? persisted.features?.dictation?.stt?.model
-      ...(voiceLocalTtsSpeed !== undefined ? { voiceLocalTtsSpeed } : {}),
+        : undefined)
-    },
+      ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
-  };
+  });
 }

packages/server/src/server/speech/speech-defaults.ts

@@ -0,0 +1,9 @@
+import {
+  DEFAULT_LOCAL_STT_MODEL,
+  DEFAULT_LOCAL_TTS_MODEL,
+} from "./providers/local/sherpa/model-catalog.js";
+
+export { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL };
+
+export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
+export const DEFAULT_OPENAI_TTS_MODEL = "tts-1";

packages/server/src/server/speech/speech-runtime.ts

@@ -29,24 +29,48 @@ import {
   DEFAULT_LOCAL_STT_MODEL,
   DEFAULT_LOCAL_TTS_MODEL,
   DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
-  type SpeechProviderId,
+  DEFAULT_OPENAI_TTS_MODEL,
-} from "./speech-types.js";
+} from "./speech-defaults.js";
+import type { SpeechProviderId } from "./speech-types.js";
 
 type LocalSttEngine =
   | { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
   | { kind: "online"; engine: SherpaOnlineRecognizerEngine };
 
-function buildModelDownloadHint(modelsDir: string, modelId: SherpaOnnxModelId): string {
+type RequestedSpeechProviders = {
-  return `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${modelsDir}' --model '${modelId}'`;
-}
-
-function resolveSpeechProviders(
-  speechConfig: PaseoSpeechConfig | null
-): {
   dictationSttProvider: SpeechProviderId;
   voiceSttProvider: SpeechProviderId;
   voiceTtsProvider: SpeechProviderId;
-} {
+};
+
+type ResolvedLocalModels = {
+  dictationLocalSttModel: LocalSttModelId;
+  voiceLocalSttModel: LocalSttModelId;
+  voiceLocalTtsModel: LocalTtsModelId;
+};
+
+type OpenAiCredentialState = {
+  openaiSttApiKey: string | undefined;
+  openaiTtsApiKey: string | undefined;
+  openaiDictationApiKey: string | undefined;
+};
+
+type InitializedLocalSpeech = {
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+  dictationSttService: SpeechToTextProvider | null;
+  localVoiceTtsProvider: TextToSpeechProvider | null;
+  localSttEngines: Map<LocalSttModelId, LocalSttEngine>;
+  requiredLocalModelIds: SherpaOnnxModelId[];
+};
+
+function buildModelDownloadHint(modelId: SherpaOnnxModelId): string {
+  return `Use 'paseo speech download --model ${modelId}' to download this model.`;
+}
+
+function resolveRequestedSpeechProviders(
+  speechConfig: PaseoSpeechConfig | null
+): RequestedSpeechProviders {
   return {
     dictationSttProvider: speechConfig?.dictationSttProvider ?? "local",
     voiceSttProvider: speechConfig?.voiceSttProvider ?? "local",
@@ -54,13 +78,9 @@ function resolveSpeechProviders(
   };
 }
 
-function resolveLocalModels(
+function resolveConfiguredLocalModels(
   speechConfig: PaseoSpeechConfig | null
-): {
+): ResolvedLocalModels {
-  dictationLocalSttModel: LocalSttModelId;
-  voiceLocalSttModel: LocalSttModelId;
-  voiceLocalTtsModel: LocalTtsModelId;
-} {
   return {
     dictationLocalSttModel: LocalSttModelIdSchema.parse(
       speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
@@ -74,12 +94,8 @@ function resolveLocalModels(
   };
 }
 
-function computeDefaultLocalModelIds(params: {
+function computeRequiredLocalModelIds(params: {
-  providers: {
+  providers: RequestedSpeechProviders;
-    dictationSttProvider: SpeechProviderId;
-    voiceSttProvider: SpeechProviderId;
-    voiceTtsProvider: SpeechProviderId;
-  };
   models: {
     dictationLocalSttModel: SherpaOnnxModelId;
     voiceLocalSttModel: SherpaOnnxModelId;
@@ -99,6 +115,50 @@ function computeDefaultLocalModelIds(params: {
   return Array.from(ids);
 }
 
+function resolveOpenAiCredentials(openaiConfig?: PaseoOpenAIConfig): OpenAiCredentialState {
+  const openaiApiKey = openaiConfig?.apiKey;
+  return {
+    openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey,
+    openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey,
+    openaiDictationApiKey: openaiApiKey,
+  };
+}
+
+function validateOpenAiCredentialRequirements(params: {
+  providers: RequestedSpeechProviders;
+  openAiCredentials: OpenAiCredentialState;
+  logger: Logger;
+}): void {
+  const { providers, openAiCredentials, logger } = params;
+  const missingOpenAiCredentialsFor: string[] = [];
+  if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) {
+    missingOpenAiCredentialsFor.push("voice.stt");
+  }
+  if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) {
+    missingOpenAiCredentialsFor.push("voice.tts");
+  }
+  if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) {
+    missingOpenAiCredentialsFor.push("dictation.stt");
+  }
+
+  if (missingOpenAiCredentialsFor.length > 0) {
+    logger.error(
+      {
+        requestedProviders: {
+          dictationStt: providers.dictationSttProvider,
+          voiceStt: providers.voiceSttProvider,
+          voiceTts: providers.voiceTtsProvider,
+        },
+        missingOpenAiCredentialsFor,
+      },
+      "Invalid speech configuration: OpenAI provider selected but credentials are missing"
+    );
+    throw new Error(
+      `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
+    );
+  }
+}
+
 async function createLocalSttEngine(params: {
   modelId: LocalSttModelId;
   modelsDir: string;
@@ -172,95 +232,25 @@ async function createLocalSttEngine(params: {
   throw new Error(`Unsupported local STT model '${modelId}'`);
 }
 
-export type InitializedSpeechRuntime = {
+async function initializeLocalSpeechServices(params: {
-  sttService: SpeechToTextProvider | null;
+  providers: RequestedSpeechProviders;
-  ttsService: TextToSpeechProvider | null;
+  localConfig: NonNullable<PaseoSpeechConfig["local"]> | null;
-  dictationSttService: SpeechToTextProvider | null;
+  localModels: ResolvedLocalModels;
-  cleanup: () => void;
+  speechConfig: PaseoSpeechConfig | null;
-  localModelConfig: {
-    modelsDir: string;
-    defaultModelIds: SherpaOnnxModelId[];
-  } | null;
-};
-
-export async function initializeSpeechRuntime(params: {
   logger: Logger;
-  openaiConfig?: PaseoOpenAIConfig;
+}): Promise<InitializedLocalSpeech> {
-  speechConfig?: PaseoSpeechConfig;
+  const { providers, localConfig, localModels, speechConfig, logger } = params;
-}): Promise<InitializedSpeechRuntime> {
-  const logger = params.logger;
-  const speechConfig = params.speechConfig ?? null;
-  const localConfig = speechConfig?.local ?? null;
-  const openaiConfig = params.openaiConfig;
 
-  const providers = resolveSpeechProviders(speechConfig);
+  const sttServices = {
-  const localModels = resolveLocalModels(speechConfig);
+    sttService: null as SpeechToTextProvider | null,
+    ttsService: null as TextToSpeechProvider | null,
+    dictationSttService: null as SpeechToTextProvider | null,
+  };
 
-  const wantsLocalDictation = providers.dictationSttProvider === "local";
+  const localSttEngines = new Map<LocalSttModelId, LocalSttEngine>();
-  const wantsLocalVoiceStt = providers.voiceSttProvider === "local";
-  const wantsLocalVoiceTts = providers.voiceTtsProvider === "local";
-
-  const openaiApiKey = openaiConfig?.apiKey;
-  const openaiSttApiKey = openaiConfig?.stt?.apiKey ?? openaiApiKey;
-  const openaiTtsApiKey = openaiConfig?.tts?.apiKey ?? openaiApiKey;
-  const openaiDictationApiKey = openaiApiKey;
-
-  const missingOpenAiCredentialsFor: string[] = [];
-  if (providers.voiceSttProvider === "openai" && !openaiSttApiKey) {
-    missingOpenAiCredentialsFor.push("voice.stt");
-  }
-  if (providers.voiceTtsProvider === "openai" && !openaiTtsApiKey) {
-    missingOpenAiCredentialsFor.push("voice.tts");
-  }
-  if (providers.dictationSttProvider === "openai" && !openaiDictationApiKey) {
-    missingOpenAiCredentialsFor.push("dictation.stt");
-  }
-  if (missingOpenAiCredentialsFor.length > 0) {
-    logger.error(
-      {
-        requestedProviders: {
-          dictationStt: providers.dictationSttProvider,
-          voiceStt: providers.voiceSttProvider,
-          voiceTts: providers.voiceTtsProvider,
-        },
-        missingOpenAiCredentialsFor,
-      },
-      "Invalid speech configuration: OpenAI provider selected but credentials are missing"
-    );
-    throw new Error(
-      `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
-    );
-  }
-
-  logger.info(
-    {
-      requestedProviders: {
-        dictationStt: providers.dictationSttProvider,
-        voiceStt: providers.voiceSttProvider,
-        voiceTts: providers.voiceTtsProvider,
-      },
-      availability: {
-        openai: {
-          stt: Boolean(openaiSttApiKey),
-          tts: Boolean(openaiTtsApiKey),
-          dictationStt: Boolean(openaiDictationApiKey),
-        },
-        local: {
-          configured: Boolean(localConfig),
-          modelsDir: localConfig?.modelsDir ?? null,
-          autoDownload: localConfig?.autoDownload ?? null,
-        },
-      },
-    },
-    "Speech provider reconciliation started"
-  );
-
-  let sttService: SpeechToTextProvider | null = null;
-  let ttsService: TextToSpeechProvider | null = null;
-  let dictationSttService: SpeechToTextProvider | null = null;
   let localVoiceTtsProvider: TextToSpeechProvider | null = null;
 
-  const requiredLocalModelIds = computeDefaultLocalModelIds({
+  const requiredLocalModelIds = computeRequiredLocalModelIds({
     providers,
     models: localModels,
   });
@@ -289,15 +279,14 @@ export async function initializeSpeechRuntime(params: {
           modelIds: requiredLocalModelIds,
           autoDownload: localConfig.autoDownload ?? true,
           hint:
-            "Run: npm run dev --workspace=@getpaseo/server, then run: " +
+            "Use `paseo speech models` to inspect status and " +
-            "`tsx packages/server/scripts/download-speech-models.ts --models-dir <DIR> --model <MODEL_ID>`",
+            "`paseo speech download --model <MODEL_ID>` to fetch missing models.",
         },
         "Failed to ensure local speech models"
       );
     }
   }
 
-  const localSttEngines = new Map<LocalSttModelId, LocalSttEngine>();
   const getLocalSttEngine = async (
     modelId: LocalSttModelId
   ): Promise<LocalSttEngine | null> => {
@@ -322,7 +311,7 @@ export async function initializeSpeechRuntime(params: {
           err,
           modelsDir: localConfig.modelsDir,
           modelId,
-          hint: buildModelDownloadHint(localConfig.modelsDir, modelId),
+          hint: buildModelDownloadHint(modelId),
         },
         "Failed to initialize local STT engine (models missing or invalid)"
       );
@@ -330,7 +319,7 @@ export async function initializeSpeechRuntime(params: {
     }
   };
 
-  if (wantsLocalVoiceStt) {
+  if (providers.voiceSttProvider === "local") {
     if (!localConfig) {
       logger.warn(
         { configured: false },
@@ -339,14 +328,14 @@ export async function initializeSpeechRuntime(params: {
     } else {
       const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel);
       if (voiceEngine?.kind === "offline") {
-        sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
+        sttServices.sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
       } else if (voiceEngine?.kind === "online") {
-        sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
+        sttServices.sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
       }
     }
   }
 
-  if (wantsLocalDictation) {
+  if (providers.dictationSttProvider === "local") {
     if (!localConfig) {
       logger.warn(
         { configured: false },
@@ -355,13 +344,13 @@ export async function initializeSpeechRuntime(params: {
     } else {
       const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel);
       if (dictationEngine?.kind === "offline") {
-        dictationSttService = {
+        sttServices.dictationSttService = {
           id: "local",
           createSession: () =>
             new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
         };
       } else if (dictationEngine?.kind === "online") {
-        dictationSttService = {
+        sttServices.dictationSttService = {
           id: "local",
           createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
         };
@@ -369,7 +358,7 @@ export async function initializeSpeechRuntime(params: {
     }
   }
 
-  if (wantsLocalVoiceTts) {
+  if (providers.voiceTtsProvider === "local") {
     if (!localConfig) {
       logger.warn(
         { configured: false },
@@ -399,14 +388,14 @@ export async function initializeSpeechRuntime(params: {
             logger
           );
         }
-        ttsService = localVoiceTtsProvider;
+        sttServices.ttsService = localVoiceTtsProvider;
       } catch (err) {
         logger.error(
           {
             err,
             modelsDir: localConfig.modelsDir,
             modelId: localModels.voiceLocalTtsModel,
-            hint: buildModelDownloadHint(localConfig.modelsDir, localModels.voiceLocalTtsModel),
+            hint: buildModelDownloadHint(localModels.voiceLocalTtsModel),
           },
           "Failed to initialize local TTS engine (models missing or invalid)"
         );
@@ -414,34 +403,65 @@ export async function initializeSpeechRuntime(params: {
     }
   }
 
+  return {
+    ...sttServices,
+    localVoiceTtsProvider,
+    localSttEngines,
+    requiredLocalModelIds,
+  };
+}
+
+function initializeOpenAiSpeechServices(params: {
+  providers: RequestedSpeechProviders;
+  openaiConfig?: PaseoOpenAIConfig;
+  openAiCredentials: OpenAiCredentialState;
+  existing: {
+    sttService: SpeechToTextProvider | null;
+    ttsService: TextToSpeechProvider | null;
+    dictationSttService: SpeechToTextProvider | null;
+  };
+  logger: Logger;
+}): {
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+  dictationSttService: SpeechToTextProvider | null;
+} {
+  const { providers, openaiConfig, openAiCredentials, existing, logger } = params;
+
+  let sttService = existing.sttService;
+  let ttsService = existing.ttsService;
+  let dictationSttService = existing.dictationSttService;
+
   const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai";
   const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai";
   const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai";
 
   if (
     (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
-    (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey)
+    (openAiCredentials.openaiSttApiKey ||
+      openAiCredentials.openaiTtsApiKey ||
+      openAiCredentials.openaiDictationApiKey)
   ) {
     logger.info("OpenAI speech provider initialized");
 
-    if (needsOpenAiStt && openaiSttApiKey) {
+    if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) {
       const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {};
       sttService = new OpenAISTT(
         {
-          apiKey: openaiSttApiKey,
+          apiKey: openAiCredentials.openaiSttApiKey,
           ...sttConfig,
         },
         logger
       );
     }
 
-    if (needsOpenAiTts && openaiTtsApiKey) {
+    if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) {
       const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {};
       ttsService = new OpenAITTS(
         {
-          apiKey: openaiTtsApiKey,
+          apiKey: openAiCredentials.openaiTtsApiKey,
           voice: "alloy",
-          model: "tts-1",
+          model: DEFAULT_OPENAI_TTS_MODEL,
           responseFormat: "pcm",
           ...ttsConfig,
         },
@@ -449,12 +469,13 @@ export async function initializeSpeechRuntime(params: {
       );
     }
 
-    if (needsOpenAiDictation && openaiDictationApiKey) {
+    const dictationApiKey = openAiCredentials.openaiDictationApiKey;
+    if (needsOpenAiDictation && dictationApiKey) {
       dictationSttService = {
         id: "openai",
         createSession: ({ logger: sessionLogger, language, prompt }) =>
           new OpenAIRealtimeTranscriptionSession({
-            apiKey: openaiDictationApiKey,
+            apiKey: dictationApiKey,
             logger: sessionLogger,
             transcriptionModel:
               openaiConfig?.realtimeTranscriptionModel
@@ -469,15 +490,101 @@ export async function initializeSpeechRuntime(params: {
     logger.warn("OpenAI speech providers are configured but credentials are missing");
   }
 
+  return {
+    sttService,
+    ttsService,
+    dictationSttService,
+  };
+}
+
+export type InitializedSpeechRuntime = {
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+  dictationSttService: SpeechToTextProvider | null;
+  cleanup: () => void;
+  localModelConfig: {
+    modelsDir: string;
+    defaultModelIds: SherpaOnnxModelId[];
+  } | null;
+};
+
+export async function initializeSpeechRuntime(params: {
+  logger: Logger;
+  openaiConfig?: PaseoOpenAIConfig;
+  speechConfig?: PaseoSpeechConfig;
+}): Promise<InitializedSpeechRuntime> {
+  const logger = params.logger;
+  const speechConfig = params.speechConfig ?? null;
+  const localConfig = speechConfig?.local ?? null;
+  const openaiConfig = params.openaiConfig;
+
+  const providers = resolveRequestedSpeechProviders(speechConfig);
+  const localModels = resolveConfiguredLocalModels(speechConfig);
+  const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
+
+  validateOpenAiCredentialRequirements({
+    providers,
+    openAiCredentials,
+    logger,
+  });
+
+  logger.info(
+    {
+      requestedProviders: {
+        dictationStt: providers.dictationSttProvider,
+        voiceStt: providers.voiceSttProvider,
+        voiceTts: providers.voiceTtsProvider,
+      },
+      availability: {
+        openai: {
+          stt: Boolean(openAiCredentials.openaiSttApiKey),
+          tts: Boolean(openAiCredentials.openaiTtsApiKey),
+          dictationStt: Boolean(openAiCredentials.openaiDictationApiKey),
+        },
+        local: {
+          configured: Boolean(localConfig),
+          modelsDir: localConfig?.modelsDir ?? null,
+          autoDownload: localConfig?.autoDownload ?? null,
+        },
+      },
+    },
+    "Speech provider reconciliation started"
+  );
+
+  const localSpeech = await initializeLocalSpeechServices({
+    providers,
+    localConfig,
+    localModels,
+    speechConfig,
+    logger,
+  });
+
+  const openAiSpeech = initializeOpenAiSpeechServices({
+    providers,
+    openaiConfig,
+    openAiCredentials,
+    existing: {
+      sttService: localSpeech.sttService,
+      ttsService: localSpeech.ttsService,
+      dictationSttService: localSpeech.dictationSttService,
+    },
+    logger,
+  });
+
   const effectiveProviders = {
-    dictationStt: dictationSttService?.id ?? "unavailable",
+    dictationStt: openAiSpeech.dictationSttService?.id ?? "unavailable",
-    voiceStt: sttService?.id ?? "unavailable",
+    voiceStt: openAiSpeech.sttService?.id ?? "unavailable",
-    voiceTts: !ttsService ? "unavailable" : ttsService === localVoiceTtsProvider ? "local" : "openai",
+    voiceTts:
+      !openAiSpeech.ttsService
+        ? "unavailable"
+        : openAiSpeech.ttsService === localSpeech.localVoiceTtsProvider
+          ? "local"
+          : "openai",
   };
   const unavailableFeatures = [
-    !dictationSttService ? "dictation.stt" : null,
+    !openAiSpeech.dictationSttService ? "dictation.stt" : null,
-    !sttService ? "voice.stt" : null,
+    !openAiSpeech.sttService ? "voice.stt" : null,
-    !ttsService ? "voice.tts" : null,
+    !openAiSpeech.ttsService ? "voice.tts" : null,
   ].filter((feature): feature is string => feature !== null);
 
   if (unavailableFeatures.length > 0) {
@@ -504,25 +611,25 @@ export async function initializeSpeechRuntime(params: {
   );
 
   const cleanup = () => {
-    const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null;
+    const maybeFreeable = localSpeech.localVoiceTtsProvider as unknown as { free?: () => void } | null;
     if (typeof maybeFreeable?.free === "function") {
       maybeFreeable.free();
     }
-    for (const engine of localSttEngines.values()) {
+    for (const engine of localSpeech.localSttEngines.values()) {
       engine.engine.free();
     }
   };
 
   return {
-    sttService,
+    sttService: openAiSpeech.sttService,
-    ttsService,
+    ttsService: openAiSpeech.ttsService,
-    dictationSttService,
+    dictationSttService: openAiSpeech.dictationSttService,
     cleanup,
     localModelConfig:
       localConfig
         ? {
             modelsDir: localConfig.modelsDir,
-            defaultModelIds: requiredLocalModelIds,
+            defaultModelIds: localSpeech.requiredLocalModelIds,
           }
         : null,
   };

packages/server/src/server/speech/speech-types.ts

@@ -1,10 +1,4 @@
-import type {
+import { z } from "zod";
-  LocalSttModelId,
-  LocalTtsModelId,
-} from "./providers/local/sherpa/model-catalog.js";
 
-export type SpeechProviderId = "openai" | "local";
+export const SpeechProviderIdSchema = z.enum(["openai", "local"]);
-
+export type SpeechProviderId = z.infer<typeof SpeechProviderIdSchema>;
-export const DEFAULT_LOCAL_STT_MODEL: LocalSttModelId = "parakeet-tdt-0.6b-v3-int8";
-export const DEFAULT_LOCAL_TTS_MODEL: LocalTtsModelId = "pocket-tts-onnx-int8";
-export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";