Update files

packages/server/src/server/bootstrap.ts

@@ -41,20 +41,9 @@ function parseListenString(listen: string): ListenTarget {
 
 import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
 import { DownloadTokenStore } from "./file-download/token-store.js";
-import { OpenAISTT, type STTConfig } from "./speech/providers/openai/stt.js";
+import type { STTConfig } from "./speech/providers/openai/stt.js";
-import { OpenAITTS, type TTSConfig } from "./speech/providers/openai/tts.js";
+import type { TTSConfig } from "./speech/providers/openai/tts.js";
-import { OpenAIRealtimeTranscriptionSession } from "./speech/providers/openai/realtime-transcription-session.js";
+import { initializeSpeechRuntime } from "./speech/speech-runtime.js";
-import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
-import { SherpaOnlineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-online-recognizer.js";
-import { SherpaOfflineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-offline-recognizer.js";
-import { SherpaOnnxSTT } from "./speech/providers/local/sherpa/sherpa-stt.js";
-import { SherpaOnnxParakeetSTT } from "./speech/providers/local/sherpa/sherpa-parakeet-stt.js";
-import { SherpaOnnxTTS } from "./speech/providers/local/sherpa/sherpa-tts.js";
-import { SherpaRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-realtime-session.js";
-import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-parakeet-realtime-session.js";
-import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/providers/local/sherpa/model-downloader.js";
-import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js";
-import { PocketTtsOnnxTTS } from "./speech/providers/local/pocket/pocket-tts-onnx.js";
 import { AgentManager } from "./agent/agent-manager.js";
 import { AgentStorage } from "./agent/agent-storage.js";
 import { attachAgentStoragePersistence } from "./persistence-hooks.js";
@@ -108,26 +97,24 @@ export type PaseoOpenAIConfig = {
   apiKey?: string;
   stt?: Partial<STTConfig> & { apiKey?: string };
   tts?: Partial<TTSConfig> & { apiKey?: string };
+  realtimeTranscriptionModel?: string;
 };
 
-export type PaseoSherpaOnnxConfig = {
+export type PaseoLocalSpeechConfig = {
   modelsDir: string;
   autoDownload?: boolean;
-  stt?: {
-    preset?: string;
-  };
-  tts?: {
-    preset?: string;
-    speakerId?: number;
-    speed?: number;
-  };
 };
 
 export type PaseoSpeechConfig = {
   dictationSttProvider?: "openai" | "local";
   voiceSttProvider?: "openai" | "local";
   voiceTtsProvider?: "openai" | "local";
-  sherpaOnnx?: PaseoSherpaOnnxConfig;
+  local?: PaseoLocalSpeechConfig;
+  dictationLocalSttModel?: string;
+  voiceLocalSttModel?: string;
+  voiceLocalTtsModel?: string;
+  voiceLocalTtsSpeakerId?: number;
+  voiceLocalTtsSpeed?: number;
 };
 
 export type PaseoDaemonConfig = {
@@ -551,384 +538,17 @@ export async function createPaseoDaemon(
       });
     },
   });
-
+  const {
-
+    sttService,
-  let sttService: SpeechToTextProvider | null = null;
+    ttsService,
-  let ttsService: TextToSpeechProvider | null = null;
+    dictationSttService,
-  let dictationSttService: SpeechToTextProvider | null = null;
+    cleanup: cleanupSpeechRuntime,
-
+    localModelConfig,
-  let sherpaOnline: SherpaOnlineRecognizerEngine | null = null;
+  } = await initializeSpeechRuntime({
-  let sherpaOffline: SherpaOfflineRecognizerEngine | null = null;
+    logger,
-  let sherpaTts: TextToSpeechProvider | null = null;
+    openaiConfig: config.openai,
-
+    speechConfig: config.speech,
-  const openaiApiKey = config.openai?.apiKey;
+  });
-  const speechConfig = config.speech ?? null;
-  const sherpaConfig = speechConfig?.sherpaOnnx ?? null;
-
-  const voiceSttProvider = speechConfig?.voiceSttProvider ?? "local";
-  const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "local";
-  const dictationSttProvider = speechConfig?.dictationSttProvider ?? "local";
-
-  const wantsLocalDictation = dictationSttProvider === "local";
-  const wantsLocalVoiceStt = voiceSttProvider === "local";
-  const wantsLocalVoiceTts = voiceTtsProvider === "local";
-
-  const openaiSttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey;
-  const openaiTtsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey;
-  const openaiDictationApiKey = openaiApiKey;
-
-  const missingOpenAiCredentialsFor: string[] = [];
-  if (voiceSttProvider === "openai" && !openaiSttApiKey) {
-    missingOpenAiCredentialsFor.push("voice.stt");
-  }
-  if (voiceTtsProvider === "openai" && !openaiTtsApiKey) {
-    missingOpenAiCredentialsFor.push("voice.tts");
-  }
-  if (dictationSttProvider === "openai" && !openaiDictationApiKey) {
-    missingOpenAiCredentialsFor.push("dictation.stt");
-  }
-
-  if (missingOpenAiCredentialsFor.length > 0) {
-    logger.error(
-      {
-        requestedProviders: {
-          dictationStt: dictationSttProvider,
-          voiceStt: voiceSttProvider,
-          voiceTts: voiceTtsProvider,
-        },
-        missingOpenAiCredentialsFor,
-      },
-      "Invalid speech configuration: OpenAI provider selected but credentials are missing"
-    );
-    throw new Error(
-      `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
-    );
-  }
-
-  logger.info(
-    {
-      requestedProviders: {
-        dictationStt: dictationSttProvider,
-        voiceStt: voiceSttProvider,
-        voiceTts: voiceTtsProvider,
-      },
-      availability: {
-        openai: {
-          stt: Boolean(openaiSttApiKey),
-          tts: Boolean(openaiTtsApiKey),
-          dictationStt: Boolean(openaiDictationApiKey),
-        },
-        local: {
-          configured: Boolean(sherpaConfig),
-          modelsDir: sherpaConfig?.modelsDir ?? null,
-          autoDownload: sherpaConfig?.autoDownload ?? null,
-        },
-      },
-    },
-    "Speech provider reconciliation started"
-  );
-
-  if ((wantsLocalDictation || wantsLocalVoiceStt || wantsLocalVoiceTts) && sherpaConfig) {
-    const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true);
-    let sttPreset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim();
-    if (
-      sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" &&
-      sttPreset !== "paraformer-bilingual-zh-en" &&
-      sttPreset !== "parakeet-tdt-0.6b-v3-int8"
-    ) {
-      throw new Error(`Unknown local STT preset: ${sttPreset}`);
-    }
-
-    let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
-    if (
-      ttsPreset !== "kitten-nano-en-v0_1-fp16" &&
-      ttsPreset !== "kokoro-en-v0_19" &&
-      ttsPreset !== "pocket-tts-onnx-int8"
-    ) {
-      throw new Error(`Unknown local TTS preset: ${ttsPreset}`);
-    }
-
-    const modelIds: SherpaOnnxModelId[] = [];
-    if (wantsLocalDictation || wantsLocalVoiceStt) {
-      modelIds.push(sttPreset as SherpaOnnxModelId);
-    }
-    if (wantsLocalVoiceTts) {
-      modelIds.push(ttsPreset as SherpaOnnxModelId);
-    }
-
-    try {
-      logger.info(
-        {
-          modelsDir: sherpaConfig.modelsDir,
-          modelIds,
-          autoDownload,
-        },
-        "Ensuring local speech models"
-      );
-      await ensureSherpaOnnxModels({
-        modelsDir: sherpaConfig.modelsDir,
-        modelIds,
-        autoDownload,
-        logger,
-      });
-    } catch (err) {
-      logger.error(
-        {
-          err,
-          modelsDir: sherpaConfig.modelsDir,
-          autoDownload,
-          hint:
-            "Run: npm run dev --workspace=@getpaseo/server, then run: " +
-            "`tsx packages/server/scripts/download-speech-models.ts --models-dir <DIR> --model <MODEL_ID>`",
-        },
-        "Failed to ensure local speech models"
-      );
-    }
-  }
-
-  if ((wantsLocalDictation || wantsLocalVoiceStt) && sherpaConfig) {
-    let preset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim();
-    if (
-      preset !== "zipformer-bilingual-zh-en-2023-02-20" &&
-      preset !== "paraformer-bilingual-zh-en" &&
-      preset !== "parakeet-tdt-0.6b-v3-int8"
-    ) {
-      throw new Error(`Unknown local STT preset: ${preset}`);
-    }
-    const base = sherpaConfig.modelsDir;
-
-    try {
-      if (preset === "parakeet-tdt-0.6b-v3-int8") {
-        const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8");
-        sherpaOffline = new SherpaOfflineRecognizerEngine(
-          {
-            model: {
-              kind: "nemo_transducer",
-              encoder: `${modelDir}/encoder.int8.onnx`,
-              decoder: `${modelDir}/decoder.int8.onnx`,
-              joiner: `${modelDir}/joiner.int8.onnx`,
-              tokens: `${modelDir}/tokens.txt`,
-            },
-            numThreads: 2,
-            debug: 0,
-          },
-          logger
-        );
-      } else {
-        const model =
-          preset === "paraformer-bilingual-zh-en"
-            ? {
-                kind: "paraformer" as const,
-                encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`,
-                decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`,
-                tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`,
-              }
-            : {
-                kind: "transducer" as const,
-                encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`,
-                decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`,
-                joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`,
-                tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`,
-                modelType: "zipformer",
-              };
-
-        sherpaOnline = new SherpaOnlineRecognizerEngine(
-          {
-            model,
-            numThreads: 1,
-            debug: 0,
-          },
-          logger
-        );
-      }
-    } catch (err) {
-      logger.error(
-        {
-          err,
-          modelsDir: sherpaConfig.modelsDir,
-          preset,
-          hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
-        },
-        "Failed to initialize Sherpa STT (models missing or invalid)"
-      );
-      sherpaOnline = null;
-      sherpaOffline = null;
-    }
-  } else if (wantsLocalDictation || wantsLocalVoiceStt) {
-    logger.warn(
-      { configured: Boolean(sherpaConfig) },
-      "Local STT selected but local provider config is missing; STT will be unavailable"
-    );
-  }
-
-  if (wantsLocalVoiceTts && sherpaConfig) {
-    let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
-    if (
-      preset !== "kitten-nano-en-v0_1-fp16" &&
-      preset !== "kokoro-en-v0_19" &&
-      preset !== "pocket-tts-onnx-int8"
-    ) {
-      throw new Error(`Unknown local TTS preset: ${preset}`);
-    }
-    try {
-      if (preset === "pocket-tts-onnx-int8") {
-        const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8");
-        sherpaTts = await PocketTtsOnnxTTS.create(
-          {
-            modelDir,
-            precision: "int8",
-            targetChunkMs: 50,
-          },
-          logger
-        );
-      } else {
-        const modelDir = `${sherpaConfig.modelsDir}/${preset}`;
-        sherpaTts = new SherpaOnnxTTS(
-          {
-            preset: preset as any,
-            modelDir,
-            speakerId: sherpaConfig.tts?.speakerId,
-            speed: sherpaConfig.tts?.speed,
-          },
-          logger
-        );
-      }
-    } catch (err) {
-      logger.error(
-        {
-          err,
-          preset,
-          hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
-        },
-        "Failed to initialize Sherpa TTS (models missing or invalid)"
-      );
-      sherpaTts = null;
-    }
-  } else if (wantsLocalVoiceTts) {
-    logger.warn(
-      { configured: Boolean(sherpaConfig) },
-      "Local TTS selected but local provider config is missing; TTS will be unavailable"
-    );
-  }
-
-  if (wantsLocalVoiceStt && sherpaOffline) {
-    sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger);
-  } else if (wantsLocalVoiceStt && sherpaOnline) {
-    sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger);
-  }
-
-  if (wantsLocalVoiceTts && sherpaTts) {
-    ttsService = sherpaTts;
-  }
-
-  if (wantsLocalDictation && sherpaOnline) {
-    dictationSttService = {
-      id: "local",
-      createSession: () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }),
-    };
-  } else if (wantsLocalDictation && sherpaOffline) {
-    dictationSttService = {
-      id: "local",
-      createSession: () =>
-        new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }),
-    };
-  }
-
-  const needsOpenAiStt = !sttService && voiceSttProvider === "openai";
-  const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai";
-  const needsOpenAiDictation = dictationSttProvider === "openai";
-
-  if (
-    (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
-    (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey)
-  ) {
-    logger.info("OpenAI speech provider initialized");
-
-    if (needsOpenAiStt) {
-      if (openaiSttApiKey) {
-        const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {};
-        sttService = new OpenAISTT(
-          {
-            apiKey: openaiSttApiKey,
-            ...sttConfig,
-          },
-          logger
-        );
-      }
-    }
-
-    if (needsOpenAiTts) {
-      if (openaiTtsApiKey) {
-        const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {};
-        ttsService = new OpenAITTS(
-          {
-            apiKey: openaiTtsApiKey,
-            voice: "alloy",
-            model: "tts-1",
-            responseFormat: "pcm",
-            ...ttsConfig,
-          },
-          logger
-        );
-      }
-    }
-
-    if (needsOpenAiDictation) {
-      const transcriptionModel =
-        process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe";
-
-      dictationSttService = {
-        id: "openai",
-        createSession: ({ logger: sessionLogger, language, prompt }) =>
-          new OpenAIRealtimeTranscriptionSession({
-            apiKey: openaiDictationApiKey!,
-            logger: sessionLogger,
-            transcriptionModel,
-            ...(language ? { language } : {}),
-            ...(prompt ? { prompt } : {}),
-            turnDetection: null,
-          }),
-      };
-    }
-  } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
-    logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation is unavailable");
-  }
-
-  const effectiveProviders = {
-    dictationStt: dictationSttService?.id ?? "unavailable",
-    voiceStt: sttService?.id ?? "unavailable",
-    voiceTts: !ttsService ? "unavailable" : ttsService === sherpaTts ? "local" : "openai",
-  };
-  const unavailableFeatures = [
-    !dictationSttService ? "dictation.stt" : null,
-    !sttService ? "voice.stt" : null,
-    !ttsService ? "voice.tts" : null,
-  ].filter((feature): feature is string => feature !== null);
-
-  if (unavailableFeatures.length > 0) {
-    logger.error(
-      {
-        requestedProviders: {
-          dictationStt: dictationSttProvider,
-          voiceStt: voiceSttProvider,
-          voiceTts: voiceTtsProvider,
-        },
-        effectiveProviders,
-        unavailableFeatures,
-      },
-      "Speech provider reconciliation failed: configured features are unavailable"
-    );
-    throw new Error(
-      `Configured speech features unavailable: ${unavailableFeatures.join(", ")}`
-    );
-  } else {
-    logger.info(
-      {
-        effectiveProviders,
-      },
-      "Speech provider reconciliation completed"
-    );
-  }
 
   wsServer = new VoiceAssistantWebSocketServer(
     httpServer,
@@ -962,6 +582,7 @@ export async function createPaseoDaemon(
     {
       finalTimeoutMs: config.dictationFinalTimeoutMs,
       stt: dictationSttService,
+      localModels: localModelConfig ?? undefined,
     }
   );
 
@@ -1064,11 +685,7 @@ export async function createPaseoDaemon(
     await agentStorage.flush().catch(() => undefined);
     await shutdownProviders(logger);
     terminalManager.killAll();
-    if (sherpaTts && typeof (sherpaTts as any).free === "function") {
+    cleanupSpeechRuntime();
-      (sherpaTts as any).free();
-    }
-    sherpaOnline?.free();
-    sherpaOffline?.free();
     await relayTransport?.stop().catch(() => undefined);
     if (wsServer) {
       await wsServer.close();

packages/server/src/server/config.ts

@@ -1,11 +1,10 @@
 import path from "node:path";
 
 import type { PaseoDaemonConfig } from "./bootstrap.js";
-import type { STTConfig } from "./speech/providers/openai/stt.js";
-import type { TTSConfig } from "./speech/providers/openai/tts.js";
 import { loadPersistedConfig } from "./persisted-config.js";
 import type { AgentProvider } from "./agent/agent-sdk-types.js";
 import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
+import { resolveSpeechConfig } from "./speech/speech-config-resolver.js";
 import {
   mergeAllowedHosts,
   parseAllowedHostsEnv,
@@ -27,69 +26,6 @@ export type CliConfigOverrides = Partial<{
   allowedHosts: AllowedHostsConfig;
 }>;
 
-function parseOpenAIConfig(
-  env: NodeJS.ProcessEnv,
-  configApiKey: string | undefined,
-  config: {
-    dictationSttModel?: string;
-    dictationSttConfidenceThreshold?: number;
-    voiceSttModel?: string;
-    voiceTtsVoice?: TTSConfig["voice"];
-    voiceTtsModel?: TTSConfig["model"];
-  }
-) {
-  const apiKey = env.OPENAI_API_KEY ?? configApiKey;
-  if (!apiKey) return undefined;
-
-  const sttConfidenceThreshold = env.STT_CONFIDENCE_THRESHOLD
-    ? parseFloat(env.STT_CONFIDENCE_THRESHOLD)
-    : config.dictationSttConfidenceThreshold;
-  const sttModel = (
-    env.STT_MODEL ??
-    config.voiceSttModel ??
-    config.dictationSttModel
-  ) as STTConfig["model"] | undefined;
-  const ttsVoice = (env.TTS_VOICE || "alloy") as
-    | "alloy"
-    | "echo"
-    | "fable"
-    | "onyx"
-    | "nova"
-    | "shimmer";
-  const ttsModel = (env.TTS_MODEL || config.voiceTtsModel || "tts-1") as
-    | "tts-1"
-    | "tts-1-hd";
-  const configuredVoice = config.voiceTtsVoice;
-
-  return {
-    apiKey,
-    stt: {
-      apiKey,
-      confidenceThreshold: sttConfidenceThreshold,
-      ...(sttModel ? { model: sttModel } : {}),
-    },
-    tts: {
-      apiKey,
-      voice: configuredVoice ?? ttsVoice,
-      model: ttsModel,
-      responseFormat: "pcm" as TTSConfig["responseFormat"],
-    },
-  };
-}
-
-function parseSpeechProviderId(value: unknown): "openai" | "local" | null {
-  if (typeof value !== "string") {
-    return null;
-  }
-  const normalized = value.trim().toLowerCase();
-  if (!normalized) {
-    return null;
-  }
-  if (normalized === "openai") return "openai";
-  if (normalized === "local") return "local";
-  return null;
-}
-
 function parseVoiceLlmProviderId(value: unknown): AgentProvider | null {
   if (typeof value !== "string") {
     return null;
@@ -103,36 +39,6 @@ function parseVoiceLlmProviderId(value: unknown): AgentProvider | null {
     : null;
 }
 
-function normalizeSherpaSttPreset(value: string): string {
-  const raw = value.trim();
-  const normalized = raw.toLowerCase();
-  if (normalized === "zipformer" || normalized === "zipformer-bilingual") {
-    return "zipformer-bilingual-zh-en-2023-02-20";
-  }
-  if (normalized === "paraformer") {
-    return "paraformer-bilingual-zh-en";
-  }
-  if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") {
-    return "parakeet-tdt-0.6b-v3-int8";
-  }
-  return raw;
-}
-
-function normalizeSherpaTtsPreset(value: string): string {
-  const raw = value.trim();
-  const normalized = raw.toLowerCase();
-  if (normalized === "pocket" || normalized === "pocket-tts") {
-    return "pocket-tts-onnx-int8";
-  }
-  if (normalized === "kitten") {
-    return "kitten-nano-en-v0_1-fp16";
-  }
-  if (normalized === "kokoro") {
-    return "kokoro-en-v0_19";
-  }
-  return raw;
-}
-
 export function loadConfig(
   paseoHome: string,
   options?: {
@@ -185,79 +91,12 @@ export function loadConfig(
   const appBaseUrl =
     env.PASEO_APP_BASE_URL ?? persisted.app?.baseUrl ?? DEFAULT_APP_BASE_URL;
 
-  const openai = parseOpenAIConfig(env, persisted.providers?.openai?.apiKey, {
+  const { openai, speech } = resolveSpeechConfig({
-    dictationSttModel: persisted.features?.dictation?.stt?.model,
+    paseoHome,
-    dictationSttConfidenceThreshold:
+    env,
-      persisted.features?.dictation?.stt?.confidenceThreshold,
+    persisted,
-    voiceSttModel: persisted.features?.voiceMode?.stt?.model,
-    voiceTtsModel: persisted.features?.voiceMode?.tts?.model,
-    voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice,
   });
 
-  const dictationSttProvider =
-    parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ??
-    parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ??
-    "local";
-
-  const voiceSttProvider =
-    parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ??
-    parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ??
-    "local";
-
-  const voiceTtsProvider =
-    parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ??
-    parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ??
-    "local";
-
-  const shouldConfigureSherpa =
-    dictationSttProvider === "local" ||
-    voiceSttProvider === "local" ||
-    voiceTtsProvider === "local" ||
-    typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" ||
-    Boolean(persisted.providers?.sherpaOnnx);
-
-  const sherpaModelsDir =
-    (env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() ||
-    path.join(paseoHome, "models", "sherpa-onnx");
-
-  const sherpaOnnx = shouldConfigureSherpa
-    ? {
-        modelsDir: sherpaModelsDir,
-        autoDownload:
-          env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined
-            ? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1"
-            : persisted.providers?.sherpaOnnx?.autoDownload ??
-              // In tests we should never hit the network unexpectedly.
-              Boolean(env.VITEST) === false,
-        stt: {
-          preset: normalizeSherpaSttPreset(
-            (env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() ||
-              (persisted.features?.voiceMode?.stt?.preset ??
-                persisted.features?.dictation?.stt?.preset)?.trim() ||
-              "parakeet-tdt-0.6b-v3-int8"
-          ),
-        },
-        tts: {
-          preset: normalizeSherpaTtsPreset(
-            (env.PASEO_SHERPA_TTS_PRESET ??
-              persisted.providers?.sherpaOnnx?.tts?.preset ??
-              persisted.features?.voiceMode?.tts?.preset)?.trim() ||
-              (env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8")
-          ),
-          speakerId:
-            env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined
-              ? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10)
-              : persisted.providers?.sherpaOnnx?.tts?.speakerId ??
-                persisted.features?.voiceMode?.tts?.speakerId,
-          speed:
-            env.PASEO_SHERPA_TTS_SPEED !== undefined
-              ? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED)
-              : persisted.providers?.sherpaOnnx?.tts?.speed ??
-                persisted.features?.voiceMode?.tts?.speed,
-        },
-      }
-    : undefined;
-
   const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER);
   const persistedVoiceLlmProvider = parseVoiceLlmProviderId(
     persisted.features?.voiceMode?.llm?.provider
@@ -284,12 +123,7 @@ export function loadConfig(
     relayPublicEndpoint,
     appBaseUrl,
     openai,
-    speech: {
+    speech,
-      dictationSttProvider,
-      voiceSttProvider,
-      voiceTtsProvider,
-      ...(sherpaOnnx ? { sherpaOnnx } : {}),
-    },
     voiceLlmProvider,
     voiceLlmProviderExplicit,
     voiceLlmModel,

packages/server/src/server/daemon-client.e2e.test.ts

@@ -18,9 +18,9 @@ import {
 
 const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
 
-const sherpaModelsDir =
+const localModelsDir =
-  process.env.PASEO_SHERPA_ONNX_MODELS_DIR ??
+  process.env.PASEO_LOCAL_MODELS_DIR ??
-  path.join(homedir(), ".paseo", "models", "sherpa-onnx");
+  path.join(homedir(), ".paseo", "models", "local-speech");
 
 function hasSherpaZipformerModels(modelsDir: string): boolean {
   return (
@@ -49,7 +49,7 @@ function hasSherpaKittenModels(modelsDir: string): boolean {
   );
 }
 
-const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir);
+const hasLocalSpeech = hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir);
 const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey);
 const speechTest = hasAnySpeech ? test : test.skip;
 
@@ -102,15 +102,17 @@ describe("daemon client E2E", () => {
           dictationSttProvider: "local" as const,
           voiceSttProvider: "local" as const,
           voiceTtsProvider: "local" as const,
-          sherpaOnnx: {
+          local: {
-            modelsDir: sherpaModelsDir,
+            modelsDir: localModelsDir,
-            stt: {
-              preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20",
-            },
-            tts: {
-              preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16",
-            },
           },
+          dictationLocalSttModel:
+            process.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
+            "zipformer-bilingual-zh-en-2023-02-20",
+          voiceLocalSttModel:
+            process.env.PASEO_VOICE_LOCAL_STT_MODEL ??
+            "zipformer-bilingual-zh-en-2023-02-20",
+          voiceLocalTtsModel:
+            process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
         };
 
     ctx = await createDaemonTestContext({

packages/server/src/server/persisted-config.ts

@@ -18,31 +18,17 @@ const ProviderCredentialsSchema = z
   })
   .strict();
 
-const SherpaOnnxProviderSchema = z
+const LocalSpeechProviderSchema = z
   .object({
     modelsDir: z.string().min(1).optional(),
     autoDownload: z.boolean().optional(),
-    stt: z
-      .object({
-        preset: z.string().min(1).optional(),
-      })
-      .strict()
-      .optional(),
-    tts: z
-      .object({
-        preset: z.string().min(1).optional(),
-        speakerId: z.number().int().optional(),
-        speed: z.number().optional(),
-      })
-      .strict()
-      .optional(),
   })
   .strict();
 
 const ProvidersSchema = z
   .object({
     openai: ProviderCredentialsSchema.optional(),
-    sherpaOnnx: SherpaOnnxProviderSchema.optional(),
+    local: LocalSpeechProviderSchema.optional(),
   })
   .strict();
 
@@ -62,7 +48,6 @@ const FeatureDictationSchema = z
       .object({
         provider: SpeechProviderIdSchema.optional(),
         model: z.string().min(1).optional(),
-        preset: z.string().min(1).optional(),
         confidenceThreshold: z.number().optional(),
       })
       .strict()
@@ -83,16 +68,14 @@ const FeatureVoiceModeSchema = z
       .object({
         provider: SpeechProviderIdSchema.optional(),
         model: z.string().min(1).optional(),
-        preset: z.string().min(1).optional(),
       })
       .strict()
       .optional(),
     tts: z
       .object({
         provider: SpeechProviderIdSchema.optional(),
-        model: z.enum(["tts-1", "tts-1-hd"]).optional(),
+        model: z.string().min(1).optional(),
         voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(),
-        preset: z.string().min(1).optional(),
         speakerId: z.number().int().optional(),
         speed: z.number().optional(),
       })

packages/server/src/server/session.ts

@@ -339,6 +339,8 @@ export class Session {
   private readonly voiceLlmProviderExplicit: boolean;
   private readonly voiceLlmModel: string | null;
   private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null;
+  private readonly localSpeechModelsDir: string;
+  private readonly defaultLocalSpeechModelIds: SherpaOnnxModelId[];
   private readonly registerVoiceSpeakHandler?: (
     agentId: string,
     handler: VoiceSpeakHandler
@@ -380,6 +382,10 @@ export class Session {
     dictation?: {
       finalTimeoutMs?: number;
       stt?: SpeechToTextProvider | null;
+      localModels?: {
+        modelsDir: string;
+        defaultModelIds: SherpaOnnxModelId[];
+      };
     }
   ) {
     this.clientId = clientId;
@@ -397,6 +403,15 @@ export class Session {
     this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false;
     this.voiceLlmModel = voice?.voiceLlmModel ?? null;
     this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null;
+    const configuredModelsDir = dictation?.localModels?.modelsDir?.trim();
+    this.localSpeechModelsDir =
+      configuredModelsDir && configuredModelsDir.length > 0
+        ? configuredModelsDir
+        : join(this.paseoHome, "models", "local-speech");
+    this.defaultLocalSpeechModelIds =
+      dictation?.localModels?.defaultModelIds && dictation.localModels.defaultModelIds.length > 0
+        ? [...new Set(dictation.localModels.defaultModelIds)]
+        : ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"];
     this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler;
     this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler;
     this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext;
@@ -1957,9 +1972,7 @@ export class Session {
   private async handleSpeechModelsListRequest(
     msg: Extract<SessionInboundMessage, { type: "speech_models_list_request" }>
   ): Promise<void> {
-    const modelsDir =
+    const modelsDir = this.localSpeechModelsDir;
-      process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() ||
-      join(this.paseoHome, "models", "sherpa-onnx");
 
     const models = await Promise.all(
       listSherpaOnnxModels().map(async (model) => {
@@ -2004,17 +2017,12 @@ export class Session {
   private async handleSpeechModelsDownloadRequest(
     msg: Extract<SessionInboundMessage, { type: "speech_models_download_request" }>
   ): Promise<void> {
-    const modelsDir =
+    const modelsDir = this.localSpeechModelsDir;
-      process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() ||
-      join(this.paseoHome, "models", "sherpa-onnx");
 
     const modelIdsRaw =
       msg.modelIds && msg.modelIds.length > 0
         ? msg.modelIds
-        : [
+        : this.defaultLocalSpeechModelIds;
-            process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20",
-            process.env.PASEO_SHERPA_TTS_PRESET ?? "pocket-tts-onnx-int8",
-          ];
 
     const allModelIds = new Set(listSherpaOnnxModels().map((m) => m.id));
     const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as SherpaOnnxModelId));

packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts

@@ -1,3 +1,5 @@
+import { z } from "zod";
+
 export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts";
 
 export type SherpaOnnxModelId =
@@ -8,6 +10,50 @@ export type SherpaOnnxModelId =
   | "kokoro-en-v0_19"
   | "pocket-tts-onnx-int8";
 
+export const LOCAL_STT_MODEL_IDS = [
+  "zipformer-bilingual-zh-en-2023-02-20",
+  "paraformer-bilingual-zh-en",
+  "parakeet-tdt-0.6b-v3-int8",
+] as const;
+export type LocalSttModelId = (typeof LOCAL_STT_MODEL_IDS)[number];
+
+export const LOCAL_TTS_MODEL_IDS = [
+  "kitten-nano-en-v0_1-fp16",
+  "kokoro-en-v0_19",
+  "pocket-tts-onnx-int8",
+] as const;
+export type LocalTtsModelId = (typeof LOCAL_TTS_MODEL_IDS)[number];
+
+const STT_MODEL_ALIASES: Record<string, (typeof LOCAL_STT_MODEL_IDS)[number]> = {
+  zipformer: "zipformer-bilingual-zh-en-2023-02-20",
+  "zipformer-bilingual": "zipformer-bilingual-zh-en-2023-02-20",
+  paraformer: "paraformer-bilingual-zh-en",
+  parakeet: "parakeet-tdt-0.6b-v3-int8",
+  "parakeet-v3": "parakeet-tdt-0.6b-v3-int8",
+  "parakeet-tdt": "parakeet-tdt-0.6b-v3-int8",
+};
+
+const TTS_MODEL_ALIASES: Record<string, (typeof LOCAL_TTS_MODEL_IDS)[number]> = {
+  pocket: "pocket-tts-onnx-int8",
+  "pocket-tts": "pocket-tts-onnx-int8",
+  kitten: "kitten-nano-en-v0_1-fp16",
+  kokoro: "kokoro-en-v0_19",
+};
+
+export const LocalSttModelIdSchema = z.preprocess((value) => {
+  if (typeof value !== "string") return value;
+  const normalized = value.trim().toLowerCase();
+  if (!normalized) return value;
+  return STT_MODEL_ALIASES[normalized] ?? normalized;
+}, z.enum(LOCAL_STT_MODEL_IDS));
+
+export const LocalTtsModelIdSchema = z.preprocess((value) => {
+  if (typeof value !== "string") return value;
+  const normalized = value.trim().toLowerCase();
+  if (!normalized) return value;
+  return TTS_MODEL_ALIASES[normalized] ?? normalized;
+}, z.enum(LOCAL_TTS_MODEL_IDS));
+
 export type SherpaOnnxModelSpec = {
   id: SherpaOnnxModelId;
   kind: SherpaOnnxModelKind;

packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts

@@ -110,7 +110,7 @@ export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOption
   if (!options.autoDownload) {
     throw new Error(
       `Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` +
-        `Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.`
+        `Set PASEO_LOCAL_AUTO_DOWNLOAD=1 to auto-download.`
     );
   }
 

packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts

@@ -96,7 +96,7 @@ describe("speech models (download E2E)", () => {
       const set = getModelSet();
 
       const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-"));
-      const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx");
+      const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech");
 
       const modelIds: SherpaOnnxModelId[] =
         set === "parakeet-pocket"
@@ -117,12 +117,20 @@ describe("speech models (download E2E)", () => {
           dictationSttProvider: "local",
           voiceSttProvider: "local",
           voiceTtsProvider: "local",
-          sherpaOnnx: {
+          local: {
             modelsDir,
             autoDownload: false,
-            stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" },
-            tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" },
           },
+          dictationLocalSttModel:
+            set === "parakeet-pocket"
+              ? "parakeet-tdt-0.6b-v3-int8"
+              : "zipformer-bilingual-zh-en-2023-02-20",
+          voiceLocalSttModel:
+            set === "parakeet-pocket"
+              ? "parakeet-tdt-0.6b-v3-int8"
+              : "zipformer-bilingual-zh-en-2023-02-20",
+          voiceLocalTtsModel:
+            set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
         },
       });
 

packages/server/src/server/speech/speech-config-resolver.test.ts

@@ -0,0 +1,94 @@
+import path from "node:path";
+
+import { describe, expect, test } from "vitest";
+
+import { PersistedConfigSchema } from "../persisted-config.js";
+import { resolveSpeechConfig } from "./speech-config-resolver.js";
+
+describe("resolveSpeechConfig", () => {
+  test("resolves local-first defaults without env overrides", () => {
+    const paseoHome = "/tmp/paseo-home";
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {} as NodeJS.ProcessEnv;
+
+    const result = resolveSpeechConfig({
+      paseoHome,
+      env,
+      persisted,
+    });
+
+    expect(result.openai).toBeUndefined();
+    expect(result.speech.dictationSttProvider).toBe("local");
+    expect(result.speech.voiceSttProvider).toBe("local");
+    expect(result.speech.voiceTtsProvider).toBe("local");
+    expect(result.speech.local).toEqual({
+      modelsDir: path.join(paseoHome, "models", "local-speech"),
+      autoDownload: true,
+    });
+    expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8");
+  });
+
+  test("resolves feature-scoped local model env vars", () => {
+    const persisted = PersistedConfigSchema.parse({
+      features: {
+        voiceMode: {
+          stt: { provider: "openai", model: "gpt-4o-transcribe" },
+        },
+      },
+      providers: {
+        openai: { apiKey: "persisted-key" },
+      },
+    });
+    const env = {
+      PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer",
+      PASEO_VOICE_LOCAL_STT_MODEL: "parakeet",
+      PASEO_VOICE_LOCAL_TTS_MODEL: "kitten",
+      PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5",
+      PASEO_VOICE_LOCAL_TTS_SPEED: "1.35",
+      PASEO_LOCAL_MODELS_DIR: "/tmp/models",
+      PASEO_LOCAL_AUTO_DOWNLOAD: "0",
+      OPENAI_API_KEY: "env-key",
+      PASEO_VOICE_STT_PROVIDER: "openai",
+      PASEO_DICTATION_STT_PROVIDER: "local",
+      PASEO_VOICE_TTS_PROVIDER: "local",
+    } as NodeJS.ProcessEnv;
+
+    const result = resolveSpeechConfig({
+      paseoHome: "/tmp/paseo-home",
+      env,
+      persisted,
+    });
+
+    expect(result.speech.local).toEqual({
+      modelsDir: "/tmp/models",
+      autoDownload: false,
+    });
+    expect(result.speech.dictationLocalSttModel).toBe("zipformer-bilingual-zh-en-2023-02-20");
+    expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.voiceLocalTtsModel).toBe("kitten-nano-en-v0_1-fp16");
+    expect(result.speech.voiceLocalTtsSpeakerId).toBe(5);
+    expect(result.speech.voiceLocalTtsSpeed).toBe(1.35);
+    expect(result.openai?.apiKey).toBe("env-key");
+    expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe");
+  });
+
+  test("ignores deprecated shared local model env vars", () => {
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {
+      PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20",
+      PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16",
+    } as NodeJS.ProcessEnv;
+
+    const result = resolveSpeechConfig({
+      paseoHome: "/tmp/paseo-home",
+      env,
+      persisted,
+    });
+
+    expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8");
+  });
+});

packages/server/src/server/speech/speech-config-resolver.ts

@@ -0,0 +1,196 @@
+import path from "node:path";
+
+import type { STTConfig } from "./providers/openai/stt.js";
+import type { TTSConfig } from "./providers/openai/tts.js";
+import type { PersistedConfig } from "../persisted-config.js";
+import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
+import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/sherpa/model-catalog.js";
+
+const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8";
+const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8";
+const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
+const DEFAULT_OPENAI_TTS_MODEL: TTSConfig["model"] = "tts-1";
+const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
+
+type SpeechProviderId = "openai" | "local";
+
+function parseSpeechProviderId(value: unknown): SpeechProviderId | null {
+  if (typeof value !== "string") return null;
+  const normalized = value.trim().toLowerCase();
+  if (!normalized) return null;
+  if (normalized === "openai") return "openai";
+  if (normalized === "local") return "local";
+  return null;
+}
+
+function parseBooleanFlag(value: string | undefined): boolean | null {
+  if (value === undefined) return null;
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "1" || normalized === "true" || normalized === "yes") return true;
+  if (normalized === "0" || normalized === "false" || normalized === "no") return false;
+  return null;
+}
+
+function parseNumberOrUndefined(value: string | undefined): number | undefined {
+  if (value === undefined) return undefined;
+  const parsed = Number.parseFloat(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+function parseIntOrUndefined(value: string | undefined): number | undefined {
+  if (value === undefined) return undefined;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+function parseOpenAiConfig(
+  env: NodeJS.ProcessEnv,
+  persisted: PersistedConfig,
+  providers: {
+    dictationSttProvider: SpeechProviderId;
+    voiceSttProvider: SpeechProviderId;
+    voiceTtsProvider: SpeechProviderId;
+  }
+): PaseoOpenAIConfig | undefined {
+  const apiKey = env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey;
+  if (!apiKey) return undefined;
+
+  const sttConfidenceThreshold = parseNumberOrUndefined(env.STT_CONFIDENCE_THRESHOLD)
+    ?? persisted.features?.dictation?.stt?.confidenceThreshold;
+
+  const sttModel = (
+    env.STT_MODEL
+    ?? (providers.voiceSttProvider === "openai" ? persisted.features?.voiceMode?.stt?.model : undefined)
+    ?? (providers.dictationSttProvider === "openai" ? persisted.features?.dictation?.stt?.model : undefined)
+  ) as STTConfig["model"] | undefined;
+
+  const ttsVoice = (
+    env.TTS_VOICE
+    || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.voice : undefined)
+    || "alloy"
+  ) as TTSConfig["voice"];
+  const ttsModelRaw =
+    env.TTS_MODEL
+    || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.model : undefined)
+    || DEFAULT_OPENAI_TTS_MODEL;
+  const ttsModel: TTSConfig["model"] =
+    ttsModelRaw === "tts-1" || ttsModelRaw === "tts-1-hd" ? ttsModelRaw : DEFAULT_OPENAI_TTS_MODEL;
+
+  const realtimeTranscriptionModel =
+    env.OPENAI_REALTIME_TRANSCRIPTION_MODEL
+    || (providers.dictationSttProvider === "openai"
+      ? persisted.features?.dictation?.stt?.model
+      : undefined)
+    || DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL;
+
+  return {
+    apiKey,
+    stt: {
+      apiKey,
+      ...(sttConfidenceThreshold !== undefined ? { confidenceThreshold: sttConfidenceThreshold } : {}),
+      ...(sttModel ? { model: sttModel } : {}),
+    },
+    tts: {
+      apiKey,
+      voice: ttsVoice,
+      model: ttsModel,
+      responseFormat: "pcm",
+    },
+    realtimeTranscriptionModel,
+  };
+}
+
+export function resolveSpeechConfig(params: {
+  paseoHome: string;
+  env: NodeJS.ProcessEnv;
+  persisted: PersistedConfig;
+}): {
+  openai: PaseoOpenAIConfig | undefined;
+  speech: PaseoSpeechConfig;
+} {
+  const { paseoHome, env, persisted } = params;
+
+  const dictationSttProvider =
+    parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER)
+    ?? parseSpeechProviderId(persisted.features?.dictation?.stt?.provider)
+    ?? "local";
+
+  const voiceSttProvider =
+    parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER)
+    ?? parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider)
+    ?? "local";
+
+  const voiceTtsProvider =
+    parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER)
+    ?? parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider)
+    ?? "local";
+
+  const anyLocalRequested =
+    dictationSttProvider === "local" ||
+    voiceSttProvider === "local" ||
+    voiceTtsProvider === "local" ||
+    env.PASEO_LOCAL_MODELS_DIR !== undefined ||
+    persisted.providers?.local !== undefined;
+
+  const localModelsDir =
+    env.PASEO_LOCAL_MODELS_DIR
+    ?? persisted.providers?.local?.modelsDir
+    ?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR);
+
+  const localAutoDownload =
+    parseBooleanFlag(env.PASEO_LOCAL_AUTO_DOWNLOAD)
+    ?? persisted.providers?.local?.autoDownload
+    ?? true;
+
+  const dictationLocalSttModel = LocalSttModelIdSchema.parse(
+    env.PASEO_DICTATION_LOCAL_STT_MODEL
+    ?? persisted.features?.dictation?.stt?.model
+    ?? DEFAULT_LOCAL_STT_MODEL
+  );
+
+  const voiceLocalSttModel = LocalSttModelIdSchema.parse(
+    env.PASEO_VOICE_LOCAL_STT_MODEL
+    ?? persisted.features?.voiceMode?.stt?.model
+    ?? DEFAULT_LOCAL_STT_MODEL
+  );
+
+  const voiceLocalTtsModel = LocalTtsModelIdSchema.parse(
+    env.PASEO_VOICE_LOCAL_TTS_MODEL
+    ?? persisted.features?.voiceMode?.tts?.model
+    ?? DEFAULT_LOCAL_TTS_MODEL
+  );
+
+  const voiceLocalTtsSpeakerId =
+    parseIntOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID)
+    ?? persisted.features?.voiceMode?.tts?.speakerId;
+
+  const voiceLocalTtsSpeed =
+    parseNumberOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEED)
+    ?? persisted.features?.voiceMode?.tts?.speed;
+
+  return {
+    openai: parseOpenAiConfig(env, persisted, {
+      dictationSttProvider,
+      voiceSttProvider,
+      voiceTtsProvider,
+    }),
+    speech: {
+      dictationSttProvider,
+      voiceSttProvider,
+      voiceTtsProvider,
+      ...(anyLocalRequested
+        ? {
+            local: {
+              modelsDir: localModelsDir.trim(),
+              autoDownload: localAutoDownload,
+            },
+          }
+        : {}),
+      dictationLocalSttModel,
+      voiceLocalSttModel,
+      voiceLocalTtsModel,
+      ...(voiceLocalTtsSpeakerId !== undefined ? { voiceLocalTtsSpeakerId } : {}),
+      ...(voiceLocalTtsSpeed !== undefined ? { voiceLocalTtsSpeed } : {}),
+    },
+  };
+}

packages/server/src/server/speech/speech-runtime.ts

@@ -0,0 +1,529 @@
+import type { Logger } from "pino";
+
+import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
+import {
+  OpenAISTT,
+} from "./providers/openai/stt.js";
+import {
+  OpenAITTS,
+} from "./providers/openai/tts.js";
+import { OpenAIRealtimeTranscriptionSession } from "./providers/openai/realtime-transcription-session.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
+import { SherpaOnlineRecognizerEngine } from "./providers/local/sherpa/sherpa-online-recognizer.js";
+import { SherpaOfflineRecognizerEngine } from "./providers/local/sherpa/sherpa-offline-recognizer.js";
+import { SherpaOnnxSTT } from "./providers/local/sherpa/sherpa-stt.js";
+import { SherpaOnnxParakeetSTT } from "./providers/local/sherpa/sherpa-parakeet-stt.js";
+import { SherpaOnnxTTS } from "./providers/local/sherpa/sherpa-tts.js";
+import { SherpaRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-realtime-session.js";
+import { SherpaParakeetRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-parakeet-realtime-session.js";
+import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./providers/local/sherpa/model-downloader.js";
+import {
+  LocalSttModelIdSchema,
+  LocalTtsModelIdSchema,
+  type LocalSttModelId,
+  type LocalTtsModelId,
+  type SherpaOnnxModelId,
+} from "./providers/local/sherpa/model-catalog.js";
+import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js";
+
+type SpeechProviderId = "openai" | "local";
+
+const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8";
+const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8";
+const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
+
+type LocalSttEngine =
+  | { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
+  | { kind: "online"; engine: SherpaOnlineRecognizerEngine };
+
+function buildModelDownloadHint(modelsDir: string, modelId: SherpaOnnxModelId): string {
+  return `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${modelsDir}' --model '${modelId}'`;
+}
+
+function resolveSpeechProviders(
+  speechConfig: PaseoSpeechConfig | null
+): {
+  dictationSttProvider: SpeechProviderId;
+  voiceSttProvider: SpeechProviderId;
+  voiceTtsProvider: SpeechProviderId;
+} {
+  return {
+    dictationSttProvider: speechConfig?.dictationSttProvider ?? "local",
+    voiceSttProvider: speechConfig?.voiceSttProvider ?? "local",
+    voiceTtsProvider: speechConfig?.voiceTtsProvider ?? "local",
+  };
+}
+
+function resolveLocalModels(
+  speechConfig: PaseoSpeechConfig | null
+): {
+  dictationLocalSttModel: LocalSttModelId;
+  voiceLocalSttModel: LocalSttModelId;
+  voiceLocalTtsModel: LocalTtsModelId;
+} {
+  return {
+    dictationLocalSttModel: LocalSttModelIdSchema.parse(
+      speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
+    ),
+    voiceLocalSttModel: LocalSttModelIdSchema.parse(
+      speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
+    ),
+    voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
+      speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL
+    ),
+  };
+}
+
+function computeDefaultLocalModelIds(params: {
+  providers: {
+    dictationSttProvider: SpeechProviderId;
+    voiceSttProvider: SpeechProviderId;
+    voiceTtsProvider: SpeechProviderId;
+  };
+  models: {
+    dictationLocalSttModel: SherpaOnnxModelId;
+    voiceLocalSttModel: SherpaOnnxModelId;
+    voiceLocalTtsModel: SherpaOnnxModelId;
+  };
+}): SherpaOnnxModelId[] {
+  const ids = new Set<SherpaOnnxModelId>();
+  if (params.providers.dictationSttProvider === "local") {
+    ids.add(params.models.dictationLocalSttModel);
+  }
+  if (params.providers.voiceSttProvider === "local") {
+    ids.add(params.models.voiceLocalSttModel);
+  }
+  if (params.providers.voiceTtsProvider === "local") {
+    ids.add(params.models.voiceLocalTtsModel);
+  }
+  return Array.from(ids);
+}
+
+async function createLocalSttEngine(params: {
+  modelId: LocalSttModelId;
+  modelsDir: string;
+  logger: Logger;
+}): Promise<LocalSttEngine> {
+  const { modelId, modelsDir, logger } = params;
+
+  if (modelId === "parakeet-tdt-0.6b-v3-int8") {
+    const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
+    return {
+      kind: "offline",
+      engine: new SherpaOfflineRecognizerEngine(
+        {
+          model: {
+            kind: "nemo_transducer",
+            encoder: `${modelDir}/encoder.int8.onnx`,
+            decoder: `${modelDir}/decoder.int8.onnx`,
+            joiner: `${modelDir}/joiner.int8.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+          },
+          numThreads: 2,
+          debug: 0,
+        },
+        logger
+      ),
+    };
+  }
+
+  if (modelId === "paraformer-bilingual-zh-en") {
+    const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
+    return {
+      kind: "online",
+      engine: new SherpaOnlineRecognizerEngine(
+        {
+          model: {
+            kind: "paraformer",
+            encoder: `${modelDir}/encoder.int8.onnx`,
+            decoder: `${modelDir}/decoder.int8.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+          },
+          numThreads: 1,
+          debug: 0,
+        },
+        logger
+      ),
+    };
+  }
+
+  if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
+    const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
+    return {
+      kind: "online",
+      engine: new SherpaOnlineRecognizerEngine(
+        {
+          model: {
+            kind: "transducer",
+            encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
+            decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
+            joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+            modelType: "zipformer",
+          },
+          numThreads: 1,
+          debug: 0,
+        },
+        logger
+      ),
+    };
+  }
+
+  throw new Error(`Unsupported local STT model '${modelId}'`);
+}
+
+export type InitializedSpeechRuntime = {
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+  dictationSttService: SpeechToTextProvider | null;
+  cleanup: () => void;
+  localModelConfig: {
+    modelsDir: string;
+    defaultModelIds: SherpaOnnxModelId[];
+  } | null;
+};
+
+export async function initializeSpeechRuntime(params: {
+  logger: Logger;
+  openaiConfig?: PaseoOpenAIConfig;
+  speechConfig?: PaseoSpeechConfig;
+}): Promise<InitializedSpeechRuntime> {
+  const logger = params.logger;
+  const speechConfig = params.speechConfig ?? null;
+  const localConfig = speechConfig?.local ?? null;
+  const openaiConfig = params.openaiConfig;
+
+  const providers = resolveSpeechProviders(speechConfig);
+  const localModels = resolveLocalModels(speechConfig);
+
+  const wantsLocalDictation = providers.dictationSttProvider === "local";
+  const wantsLocalVoiceStt = providers.voiceSttProvider === "local";
+  const wantsLocalVoiceTts = providers.voiceTtsProvider === "local";
+
+  const openaiApiKey = openaiConfig?.apiKey;
+  const openaiSttApiKey = openaiConfig?.stt?.apiKey ?? openaiApiKey;
+  const openaiTtsApiKey = openaiConfig?.tts?.apiKey ?? openaiApiKey;
+  const openaiDictationApiKey = openaiApiKey;
+
+  const missingOpenAiCredentialsFor: string[] = [];
+  if (providers.voiceSttProvider === "openai" && !openaiSttApiKey) {
+    missingOpenAiCredentialsFor.push("voice.stt");
+  }
+  if (providers.voiceTtsProvider === "openai" && !openaiTtsApiKey) {
+    missingOpenAiCredentialsFor.push("voice.tts");
+  }
+  if (providers.dictationSttProvider === "openai" && !openaiDictationApiKey) {
+    missingOpenAiCredentialsFor.push("dictation.stt");
+  }
+  if (missingOpenAiCredentialsFor.length > 0) {
+    logger.error(
+      {
+        requestedProviders: {
+          dictationStt: providers.dictationSttProvider,
+          voiceStt: providers.voiceSttProvider,
+          voiceTts: providers.voiceTtsProvider,
+        },
+        missingOpenAiCredentialsFor,
+      },
+      "Invalid speech configuration: OpenAI provider selected but credentials are missing"
+    );
+    throw new Error(
+      `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
+    );
+  }
+
+  logger.info(
+    {
+      requestedProviders: {
+        dictationStt: providers.dictationSttProvider,
+        voiceStt: providers.voiceSttProvider,
+        voiceTts: providers.voiceTtsProvider,
+      },
+      availability: {
+        openai: {
+          stt: Boolean(openaiSttApiKey),
+          tts: Boolean(openaiTtsApiKey),
+          dictationStt: Boolean(openaiDictationApiKey),
+        },
+        local: {
+          configured: Boolean(localConfig),
+          modelsDir: localConfig?.modelsDir ?? null,
+          autoDownload: localConfig?.autoDownload ?? null,
+        },
+      },
+    },
+    "Speech provider reconciliation started"
+  );
+
+  let sttService: SpeechToTextProvider | null = null;
+  let ttsService: TextToSpeechProvider | null = null;
+  let dictationSttService: SpeechToTextProvider | null = null;
+  let localVoiceTtsProvider: TextToSpeechProvider | null = null;
+
+  const requiredLocalModelIds = computeDefaultLocalModelIds({
+    providers,
+    models: localModels,
+  });
+
+  if (requiredLocalModelIds.length > 0 && localConfig) {
+    try {
+      logger.info(
+        {
+          modelsDir: localConfig.modelsDir,
+          modelIds: requiredLocalModelIds,
+          autoDownload: localConfig.autoDownload ?? true,
+        },
+        "Ensuring local speech models"
+      );
+      await ensureSherpaOnnxModels({
+        modelsDir: localConfig.modelsDir,
+        modelIds: requiredLocalModelIds,
+        autoDownload: localConfig.autoDownload ?? true,
+        logger,
+      });
+    } catch (err) {
+      logger.error(
+        {
+          err,
+          modelsDir: localConfig.modelsDir,
+          modelIds: requiredLocalModelIds,
+          autoDownload: localConfig.autoDownload ?? true,
+          hint:
+            "Run: npm run dev --workspace=@getpaseo/server, then run: " +
+            "`tsx packages/server/scripts/download-speech-models.ts --models-dir <DIR> --model <MODEL_ID>`",
+        },
+        "Failed to ensure local speech models"
+      );
+    }
+  }
+
+  const localSttEngines = new Map<LocalSttModelId, LocalSttEngine>();
+  const getLocalSttEngine = async (
+    modelId: LocalSttModelId
+  ): Promise<LocalSttEngine | null> => {
+    const existing = localSttEngines.get(modelId);
+    if (existing) {
+      return existing;
+    }
+    if (!localConfig) {
+      return null;
+    }
+    try {
+      const created = await createLocalSttEngine({
+        modelId,
+        modelsDir: localConfig.modelsDir,
+        logger,
+      });
+      localSttEngines.set(modelId, created);
+      return created;
+    } catch (err) {
+      logger.error(
+        {
+          err,
+          modelsDir: localConfig.modelsDir,
+          modelId,
+          hint: buildModelDownloadHint(localConfig.modelsDir, modelId),
+        },
+        "Failed to initialize local STT engine (models missing or invalid)"
+      );
+      return null;
+    }
+  };
+
+  if (wantsLocalVoiceStt) {
+    if (!localConfig) {
+      logger.warn(
+        { configured: false },
+        "Local STT selected for voice but local provider config is missing; STT will be unavailable"
+      );
+    } else {
+      const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel);
+      if (voiceEngine?.kind === "offline") {
+        sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
+      } else if (voiceEngine?.kind === "online") {
+        sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
+      }
+    }
+  }
+
+  if (wantsLocalDictation) {
+    if (!localConfig) {
+      logger.warn(
+        { configured: false },
+        "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable"
+      );
+    } else {
+      const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel);
+      if (dictationEngine?.kind === "offline") {
+        dictationSttService = {
+          id: "local",
+          createSession: () =>
+            new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
+        };
+      } else if (dictationEngine?.kind === "online") {
+        dictationSttService = {
+          id: "local",
+          createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
+        };
+      }
+    }
+  }
+
+  if (wantsLocalVoiceTts) {
+    if (!localConfig) {
+      logger.warn(
+        { configured: false },
+        "Local TTS selected for voice but local provider config is missing; TTS will be unavailable"
+      );
+    } else {
+      try {
+        if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
+          const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
+          localVoiceTtsProvider = await PocketTtsOnnxTTS.create(
+            {
+              modelDir,
+              precision: "int8",
+              targetChunkMs: 50,
+            },
+            logger
+          );
+        } else {
+          const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
+          localVoiceTtsProvider = new SherpaOnnxTTS(
+            {
+              preset: localModels.voiceLocalTtsModel,
+              modelDir,
+              speakerId: speechConfig?.voiceLocalTtsSpeakerId,
+              speed: speechConfig?.voiceLocalTtsSpeed,
+            },
+            logger
+          );
+        }
+        ttsService = localVoiceTtsProvider;
+      } catch (err) {
+        logger.error(
+          {
+            err,
+            modelsDir: localConfig.modelsDir,
+            modelId: localModels.voiceLocalTtsModel,
+            hint: buildModelDownloadHint(localConfig.modelsDir, localModels.voiceLocalTtsModel),
+          },
+          "Failed to initialize local TTS engine (models missing or invalid)"
+        );
+      }
+    }
+  }
+
+  const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai";
+  const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai";
+  const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai";
+
+  if (
+    (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
+    (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey)
+  ) {
+    logger.info("OpenAI speech provider initialized");
+
+    if (needsOpenAiStt && openaiSttApiKey) {
+      const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {};
+      sttService = new OpenAISTT(
+        {
+          apiKey: openaiSttApiKey,
+          ...sttConfig,
+        },
+        logger
+      );
+    }
+
+    if (needsOpenAiTts && openaiTtsApiKey) {
+      const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {};
+      ttsService = new OpenAITTS(
+        {
+          apiKey: openaiTtsApiKey,
+          voice: "alloy",
+          model: "tts-1",
+          responseFormat: "pcm",
+          ...ttsConfig,
+        },
+        logger
+      );
+    }
+
+    if (needsOpenAiDictation && openaiDictationApiKey) {
+      dictationSttService = {
+        id: "openai",
+        createSession: ({ logger: sessionLogger, language, prompt }) =>
+          new OpenAIRealtimeTranscriptionSession({
+            apiKey: openaiDictationApiKey,
+            logger: sessionLogger,
+            transcriptionModel:
+              openaiConfig?.realtimeTranscriptionModel
+              ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+            ...(language ? { language } : {}),
+            ...(prompt ? { prompt } : {}),
+            turnDetection: null,
+          }),
+      };
+    }
+  } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
+    logger.warn("OpenAI speech providers are configured but credentials are missing");
+  }
+
+  const effectiveProviders = {
+    dictationStt: dictationSttService?.id ?? "unavailable",
+    voiceStt: sttService?.id ?? "unavailable",
+    voiceTts: !ttsService ? "unavailable" : ttsService === localVoiceTtsProvider ? "local" : "openai",
+  };
+  const unavailableFeatures = [
+    !dictationSttService ? "dictation.stt" : null,
+    !sttService ? "voice.stt" : null,
+    !ttsService ? "voice.tts" : null,
+  ].filter((feature): feature is string => feature !== null);
+
+  if (unavailableFeatures.length > 0) {
+    logger.error(
+      {
+        requestedProviders: {
+          dictationStt: providers.dictationSttProvider,
+          voiceStt: providers.voiceSttProvider,
+          voiceTts: providers.voiceTtsProvider,
+        },
+        effectiveProviders,
+        unavailableFeatures,
+      },
+      "Speech provider reconciliation failed: configured features are unavailable"
+    );
+    throw new Error(`Configured speech features unavailable: ${unavailableFeatures.join(", ")}`);
+  }
+
+  logger.info(
+    {
+      effectiveProviders,
+    },
+    "Speech provider reconciliation completed"
+  );
+
+  const cleanup = () => {
+    const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null;
+    if (typeof maybeFreeable?.free === "function") {
+      maybeFreeable.free();
+    }
+    for (const engine of localSttEngines.values()) {
+      engine.engine.free();
+    }
+  };
+
+  return {
+    sttService,
+    ttsService,
+    dictationSttService,
+    cleanup,
+    localModelConfig:
+      localConfig
+        ? {
+            modelsDir: localConfig.modelsDir,
+            defaultModelIds: requiredLocalModelIds,
+          }
+        : null,
+  };
+}

packages/server/src/server/websocket-server.ts

@@ -20,6 +20,7 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js";
 import { PushTokenStore } from "./push/token-store.js";
 import { PushService } from "./push/push-service.js";
 import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
+import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js";
 
 export type AgentMcpTransportFactory = () => Promise<Transport>;
 type VoiceMcpStdioConfig = {
@@ -76,6 +77,10 @@ export class VoiceAssistantWebSocketServer {
   private readonly dictation: {
     finalTimeoutMs?: number;
     stt?: SpeechToTextProvider | null;
+    localModels?: {
+      modelsDir: string;
+      defaultModelIds: SherpaOnnxModelId[];
+    };
   } | null;
   private readonly voice: {
     voiceLlmProvider?: AgentProvider | null;
@@ -120,6 +125,10 @@ export class VoiceAssistantWebSocketServer {
     dictation?: {
       finalTimeoutMs?: number;
       stt?: SpeechToTextProvider | null;
+      localModels?: {
+        modelsDir: string;
+        defaultModelIds: SherpaOnnxModelId[];
+      };
     }
   ) {
     this.logger = logger.child({ module: "websocket-server" });

packages/website/public/schemas/paseo.config.v1.json

@@ -86,7 +86,7 @@
               },
               "additionalProperties": false
             },
-            "sherpaOnnx": {
+            "local": {
               "type": "object",
               "properties": {
                 "modelsDir": {
@@ -95,32 +95,6 @@
                 },
                 "autoDownload": {
                   "type": "boolean"
-                },
-                "stt": {
-                  "type": "object",
-                  "properties": {
-                    "preset": {
-                      "type": "string",
-                      "minLength": 1
-                    }
-                  },
-                  "additionalProperties": false
-                },
-                "tts": {
-                  "type": "object",
-                  "properties": {
-                    "preset": {
-                      "type": "string",
-                      "minLength": 1
-                    },
-                    "speakerId": {
-                      "type": "number"
-                    },
-                    "speed": {
-                      "type": "number"
-                    }
-                  },
-                  "additionalProperties": false
                 }
               },
               "additionalProperties": false
@@ -207,10 +181,7 @@
                     },
                     "model": {
                       "type": "string",
-                      "enum": [
+                      "minLength": 1
-                        "tts-1",
-                        "tts-1-hd"
-                      ]
                     },
                     "voice": {
                       "type": "string",

packages/website/src/routeTree.gen.ts

@@ -13,6 +13,7 @@ import { Route as DocsRouteImport } from './routes/docs'
 import { Route as IndexRouteImport } from './routes/index'
 import { Route as DocsIndexRouteImport } from './routes/docs/index'
 import { Route as DocsWorktreesRouteImport } from './routes/docs/worktrees'
+import { Route as DocsVoiceRouteImport } from './routes/docs/voice'
 import { Route as DocsSecurityRouteImport } from './routes/docs/security'
 import { Route as DocsConfigurationRouteImport } from './routes/docs/configuration'
 import { Route as DocsCliRouteImport } from './routes/docs/cli'
@@ -38,6 +39,11 @@ const DocsWorktreesRoute = DocsWorktreesRouteImport.update({
   path: '/worktrees',
   getParentRoute: () => DocsRoute,
 } as any)
+const DocsVoiceRoute = DocsVoiceRouteImport.update({
+  id: '/voice',
+  path: '/voice',
+  getParentRoute: () => DocsRoute,
+} as any)
 const DocsSecurityRoute = DocsSecurityRouteImport.update({
   id: '/security',
   path: '/security',
@@ -66,6 +72,7 @@ export interface FileRoutesByFullPath {
   '/docs/cli': typeof DocsCliRoute
   '/docs/configuration': typeof DocsConfigurationRoute
   '/docs/security': typeof DocsSecurityRoute
+  '/docs/voice': typeof DocsVoiceRoute
   '/docs/worktrees': typeof DocsWorktreesRoute
   '/docs/': typeof DocsIndexRoute
 }
@@ -75,6 +82,7 @@ export interface FileRoutesByTo {
   '/docs/cli': typeof DocsCliRoute
   '/docs/configuration': typeof DocsConfigurationRoute
   '/docs/security': typeof DocsSecurityRoute
+  '/docs/voice': typeof DocsVoiceRoute
   '/docs/worktrees': typeof DocsWorktreesRoute
   '/docs': typeof DocsIndexRoute
 }
@@ -86,6 +94,7 @@ export interface FileRoutesById {
   '/docs/cli': typeof DocsCliRoute
   '/docs/configuration': typeof DocsConfigurationRoute
   '/docs/security': typeof DocsSecurityRoute
+  '/docs/voice': typeof DocsVoiceRoute
   '/docs/worktrees': typeof DocsWorktreesRoute
   '/docs/': typeof DocsIndexRoute
 }
@@ -98,6 +107,7 @@ export interface FileRouteTypes {
     | '/docs/cli'
     | '/docs/configuration'
     | '/docs/security'
+    | '/docs/voice'
     | '/docs/worktrees'
     | '/docs/'
   fileRoutesByTo: FileRoutesByTo
@@ -107,6 +117,7 @@ export interface FileRouteTypes {
     | '/docs/cli'
     | '/docs/configuration'
     | '/docs/security'
+    | '/docs/voice'
     | '/docs/worktrees'
     | '/docs'
   id:
@@ -117,6 +128,7 @@ export interface FileRouteTypes {
     | '/docs/cli'
     | '/docs/configuration'
     | '/docs/security'
+    | '/docs/voice'
     | '/docs/worktrees'
     | '/docs/'
   fileRoutesById: FileRoutesById
@@ -156,6 +168,13 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof DocsWorktreesRouteImport
       parentRoute: typeof DocsRoute
     }
+    '/docs/voice': {
+      id: '/docs/voice'
+      path: '/voice'
+      fullPath: '/docs/voice'
+      preLoaderRoute: typeof DocsVoiceRouteImport
+      parentRoute: typeof DocsRoute
+    }
     '/docs/security': {
       id: '/docs/security'
       path: '/security'
@@ -192,6 +211,7 @@ interface DocsRouteChildren {
   DocsCliRoute: typeof DocsCliRoute
   DocsConfigurationRoute: typeof DocsConfigurationRoute
   DocsSecurityRoute: typeof DocsSecurityRoute
+  DocsVoiceRoute: typeof DocsVoiceRoute
   DocsWorktreesRoute: typeof DocsWorktreesRoute
   DocsIndexRoute: typeof DocsIndexRoute
 }
@@ -201,6 +221,7 @@ const DocsRouteChildren: DocsRouteChildren = {
   DocsCliRoute: DocsCliRoute,
   DocsConfigurationRoute: DocsConfigurationRoute,
   DocsSecurityRoute: DocsSecurityRoute,
+  DocsVoiceRoute: DocsVoiceRoute,
   DocsWorktreesRoute: DocsWorktreesRoute,
   DocsIndexRoute: DocsIndexRoute,
 }

packages/website/src/routes/docs.tsx

@@ -7,6 +7,7 @@ export const Route = createFileRoute('/docs')({
 
 const navigation = [
   { name: 'Getting started', href: '/docs' },
+  { name: 'Voice', href: '/docs/voice' },
   { name: 'Git worktrees', href: '/docs/worktrees' },
   { name: 'CLI', href: '/docs/cli' },
   { name: 'Configuration', href: '/docs/configuration' },

packages/website/src/routes/docs/configuration.tsx

@@ -81,75 +81,13 @@ function Configuration() {
       <section className="space-y-4">
         <h2 className="text-xl font-medium">Voice</h2>
         <p className="text-white/60 leading-relaxed">
-          Voice is provider-based per feature: dictation STT, realtime voice STT, and voice TTS can each
+          Voice is configured through <code className="font-mono">features.dictation</code> and{' '}
-          use <code className="font-mono">local</code> or <code className="font-mono">openai</code>.
+          <code className="font-mono">features.voiceMode</code>, with provider credentials under{' '}
-          Defaults are local for all three.
+          <code className="font-mono">providers</code>.
-        </p>
-        <pre className="bg-card border border-border rounded-lg p-4 font-mono text-sm overflow-x-auto text-white/80">
-{`{
-  "version": 1,
-  "features": {
-    "dictation": { "stt": { "provider": "local" } },
-    "voiceMode": {
-      "llm": { "provider": "claude", "model": "haiku" },
-      "stt": { "provider": "local" },
-      "tts": { "provider": "local" }
-    }
-  },
-  "providers": {
-    "sherpaOnnx": {
-      "modelsDir": "~/.paseo/models/sherpa-onnx",
-      "autoDownload": true,
-      "stt": { "preset": "parakeet-tdt-0.6b-v3-int8" },
-      "tts": { "preset": "pocket-tts-onnx-int8" }
-    }
-  }
-}`}
-        </pre>
-        <p className="text-white/60 leading-relaxed">
-          Voice LLM orchestration is agents-only. Set{' '}
-          <code className="font-mono">features.voiceMode.llm.provider</code> to one of{' '}
-          <code className="font-mono">claude</code>, <code className="font-mono">codex</code>, or{' '}
-          <code className="font-mono">opencode</code>. If set, startup is strict and fails if unavailable.
-          If omitted, Paseo picks the first available voice-enabled agent provider.
         </p>
         <p className="text-white/60 leading-relaxed">
-          Local voice uses ONNX models (sherpa-onnx + PocketTTS). Default presets are
+          For voice philosophy, architecture, and complete local/OpenAI setup examples, see{' '}
-          <code className="font-mono">parakeet-tdt-0.6b-v3-int8</code> for STT and
+          <a href="/docs/voice" className="underline hover:text-white/80">Voice docs</a>.
-          <code className="font-mono">pocket-tts-onnx-int8</code> for TTS.
-        </p>
-        <p className="text-white/60 leading-relaxed">
-          With local provider enabled and auto-download on (default outside tests), missing model files are
-          downloaded at daemon startup into <code className="font-mono">$PASEO_HOME/models/sherpa-onnx</code>.
-          Downloads are only performed for missing files.
-        </p>
-        <p className="text-white/60 leading-relaxed">
-          Local ONNX execution is CPU-first by default. It works without GPU requirements, but performance
-          depends on your machine.
-        </p>
-        <p className="text-white/60 leading-relaxed">
-          To force OpenAI for voice/dictation, configure OpenAI providers explicitly and provide
-          <code className="font-mono">OPENAI_API_KEY</code>. If OpenAI is selected but credentials are
-          missing, daemon startup fails fast.
-        </p>
-        <pre className="bg-card border border-border rounded-lg p-4 font-mono text-sm overflow-x-auto text-white/80">
-{`{
-  "version": 1,
-  "features": {
-    "dictation": { "stt": { "provider": "openai" } },
-    "voiceMode": {
-      "stt": { "provider": "openai" },
-      "tts": { "provider": "openai" }
-    }
-  },
-  "providers": {
-    "openai": { "apiKey": "..." }
-  }
-}`}
-        </pre>
-        <p className="text-white/60 leading-relaxed">
-          Realtime voice can create/manage coding agents. Use clear prompts and explicit paths, because
-          those agents can be launched in arbitrary working directories by request.
         </p>
       </section>
 
@@ -162,8 +100,10 @@ function Configuration() {
           <li><code className="font-mono">OPENAI_API_KEY</code> — override OpenAI provider key</li>
           <li><code className="font-mono">PASEO_VOICE_LLM_PROVIDER</code> — override voice LLM provider (<code className="font-mono">claude</code>, <code className="font-mono">codex</code>, <code className="font-mono">opencode</code>)</li>
           <li><code className="font-mono">PASEO_DICTATION_STT_PROVIDER</code>, <code className="font-mono">PASEO_VOICE_STT_PROVIDER</code>, <code className="font-mono">PASEO_VOICE_TTS_PROVIDER</code> — override voice provider selection (<code className="font-mono">local</code> or <code className="font-mono">openai</code>)</li>
-          <li><code className="font-mono">PASEO_SHERPA_ONNX_MODELS_DIR</code> and <code className="font-mono">PASEO_SHERPA_ONNX_AUTO_DOWNLOAD</code> — control local model directory and download behavior</li>
+          <li><code className="font-mono">PASEO_LOCAL_MODELS_DIR</code> and <code className="font-mono">PASEO_LOCAL_AUTO_DOWNLOAD</code> — control local model directory and download behavior</li>
-          <li><code className="font-mono">PASEO_SHERPA_STT_PRESET</code> and <code className="font-mono">PASEO_SHERPA_TTS_PRESET</code> — override local STT/TTS model presets</li>
+          <li><code className="font-mono">PASEO_DICTATION_LOCAL_STT_MODEL</code> — override local dictation STT model</li>
+          <li><code className="font-mono">PASEO_VOICE_LOCAL_STT_MODEL</code>, <code className="font-mono">PASEO_VOICE_LOCAL_TTS_MODEL</code> — override local voice STT/TTS models</li>
+          <li><code className="font-mono">PASEO_VOICE_LOCAL_TTS_SPEAKER_ID</code>, <code className="font-mono">PASEO_VOICE_LOCAL_TTS_SPEED</code> — optional local voice TTS tuning</li>
         </ul>
       </section>
 

packages/website/src/routes/docs/index.tsx

@@ -87,22 +87,25 @@ function GettingStarted() {
       <section className="space-y-4">
         <h2 className="text-xl font-medium">Voice Setup</h2>
         <p className="text-white/60">
-          Voice supports two providers: local (default) and OpenAI. Local uses ONNX models and will
+          Paseo includes first-class voice support with a local-first architecture and configurable speech
-          download missing model files automatically at daemon startup.
+          providers.
         </p>
-        <div className="bg-card border border-border rounded-lg p-4 font-mono text-sm">
-          <span className="text-muted-foreground select-none">$ </span>
-          <span>export OPENAI_API_KEY=your-key-here</span>
-        </div>
         <p className="text-white/60">
-          Set <code className="font-mono">OPENAI_API_KEY</code> if you want OpenAI speech providers.
+          For architecture, local model behavior, and provider configuration, see the Voice docs page.
-          For local defaults and model presets, see Configuration.
         </p>
+        <a href="/docs/voice" className="underline hover:text-white/80">
+          Voice docs
+        </a>
       </section>
 
       <section className="space-y-4">
         <h2 className="text-xl font-medium">Next</h2>
         <ul className="text-white/60 space-y-2 list-disc list-inside">
+          <li>
+            <a href="/docs/voice" className="underline hover:text-white/80">
+              Voice
+            </a>
+          </li>
           <li>
             <a href="/docs/configuration" className="underline hover:text-white/80">
               Configuration

packages/website/src/routes/docs/voice.tsx

@@ -0,0 +1,126 @@
+import { createFileRoute } from '@tanstack/react-router'
+
+export const Route = createFileRoute('/docs/voice')({
+  head: () => ({
+    meta: [
+      { title: 'Voice - Paseo Docs' },
+      {
+        name: 'description',
+        content: 'Paseo voice architecture, local-first model execution, and provider configuration.',
+      },
+    ],
+  }),
+  component: VoiceDocs,
+})
+
+function VoiceDocs() {
+  return (
+    <div className="space-y-8">
+      <div>
+        <h1 className="text-3xl font-medium font-title mb-4">Voice</h1>
+        <p className="text-white/60 leading-relaxed">
+          Paseo has first-class voice support for dictation and realtime conversations with your coding
+          environment.
+        </p>
+      </div>
+
+      <section className="space-y-4">
+        <h2 className="text-xl font-medium">Philosophy</h2>
+        <p className="text-white/60 leading-relaxed">
+          Voice is local-first. You can run speech fully on-device, or choose OpenAI for speech features.
+          For voice reasoning/orchestration, Paseo reuses agent providers already installed and authenticated
+          on your machine.
+        </p>
+        <p className="text-white/60 leading-relaxed">
+          This keeps credentials and execution in your environment and avoids introducing a separate
+          cloud-only voice stack.
+        </p>
+      </section>
+
+      <section className="space-y-4">
+        <h2 className="text-xl font-medium">Architecture</h2>
+        <ul className="text-white/60 space-y-2 list-disc list-inside">
+          <li>Speech I/O: STT and TTS providers per feature (<code className="font-mono">local</code> or <code className="font-mono">openai</code>)</li>
+          <li>Local speech runtime: ONNX models executed on CPU by default</li>
+          <li>Voice LLM orchestration: hidden agent session using your configured provider (<code className="font-mono">claude</code>, <code className="font-mono">codex</code>, or <code className="font-mono">opencode</code>)</li>
+          <li>Tooling path: MCP stdio bridge for voice tools and agent control</li>
+        </ul>
+      </section>
+
+      <section className="space-y-4">
+        <h2 className="text-xl font-medium">Local Speech</h2>
+        <p className="text-white/60 leading-relaxed">
+          Local speech defaults to model IDs <code className="font-mono">parakeet-tdt-0.6b-v3-int8</code>{' '}
+          (STT) and <code className="font-mono">pocket-tts-onnx-int8</code> (TTS).
+        </p>
+        <p className="text-white/60 leading-relaxed">
+          Missing models are downloaded at daemon startup into{' '}
+          <code className="font-mono">$PASEO_HOME/models/local-speech</code> when auto-download is enabled.
+          Downloads happen only for missing files.
+        </p>
+        <pre className="bg-card border border-border rounded-lg p-4 font-mono text-sm overflow-x-auto text-white/80">
+{`{
+  "version": 1,
+  "features": {
+    "dictation": { "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8" } },
+    "voiceMode": {
+      "llm": { "provider": "claude", "model": "haiku" },
+      "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8" },
+      "tts": { "provider": "local", "model": "pocket-tts-onnx-int8" }
+    }
+  },
+  "providers": {
+    "local": {
+      "modelsDir": "~/.paseo/models/local-speech",
+      "autoDownload": true
+    }
+  }
+}`}
+        </pre>
+      </section>
+
+      <section className="space-y-4">
+        <h2 className="text-xl font-medium">OpenAI Speech Option</h2>
+        <p className="text-white/60 leading-relaxed">
+          You can switch dictation, voice STT, and voice TTS to OpenAI by setting provider fields to{' '}
+          <code className="font-mono">openai</code> and providing <code className="font-mono">OPENAI_API_KEY</code>.
+        </p>
+        <pre className="bg-card border border-border rounded-lg p-4 font-mono text-sm overflow-x-auto text-white/80">
+{`{
+  "version": 1,
+  "features": {
+    "dictation": { "stt": { "provider": "openai" } },
+    "voiceMode": {
+      "stt": { "provider": "openai" },
+      "tts": { "provider": "openai" }
+    }
+  },
+  "providers": {
+    "openai": { "apiKey": "..." }
+  }
+}`}
+        </pre>
+      </section>
+
+      <section className="space-y-4">
+        <h2 className="text-xl font-medium">Environment Variables</h2>
+        <ul className="text-white/60 space-y-2 list-disc list-inside">
+          <li><code className="font-mono">OPENAI_API_KEY</code> — OpenAI speech credentials</li>
+          <li><code className="font-mono">PASEO_VOICE_LLM_PROVIDER</code> — voice agent provider override</li>
+          <li><code className="font-mono">PASEO_LOCAL_MODELS_DIR</code>, <code className="font-mono">PASEO_LOCAL_AUTO_DOWNLOAD</code> — local model storage and download policy</li>
+          <li><code className="font-mono">PASEO_DICTATION_LOCAL_STT_MODEL</code> — local dictation STT model ID</li>
+          <li><code className="font-mono">PASEO_VOICE_LOCAL_STT_MODEL</code>, <code className="font-mono">PASEO_VOICE_LOCAL_TTS_MODEL</code> — local voice STT/TTS model IDs</li>
+          <li><code className="font-mono">PASEO_VOICE_LOCAL_TTS_SPEAKER_ID</code>, <code className="font-mono">PASEO_VOICE_LOCAL_TTS_SPEED</code> — optional local voice TTS tuning</li>
+        </ul>
+      </section>
+
+      <section className="space-y-4">
+        <h2 className="text-xl font-medium">Operational Notes</h2>
+        <p className="text-white/60 leading-relaxed">
+          Realtime voice can launch and control agents. Treat voice prompts with the same care as direct
+          agent instructions, especially when specifying working directories or destructive operations.
+        </p>
+      </section>
+    </div>
+  )
+}