speech: move defaults and runtime into provider modules

packages/server/src/server/bootstrap.ts

@@ -41,8 +41,8 @@ function parseListenString(listen: string): ListenTarget {
 
 import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
 import { DownloadTokenStore } from "./file-download/token-store.js";
-import type { STTConfig } from "./speech/providers/openai/stt.js";
+import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js";
-import type { TTSConfig } from "./speech/providers/openai/tts.js";
+import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js";
 import { initializeSpeechRuntime } from "./speech/speech-runtime.js";
 import { AgentManager } from "./agent/agent-manager.js";
 import { AgentStorage } from "./agent/agent-storage.js";
@@ -101,17 +101,8 @@ function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseAr
   return resolved;
 }
 
-export type PaseoOpenAIConfig = {
+export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig;
-  apiKey?: string;
+export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig;
-  stt?: Partial<STTConfig> & { apiKey?: string };
-  tts?: Partial<TTSConfig> & { apiKey?: string };
-  realtimeTranscriptionModel?: string;
-};
-
-export type PaseoLocalSpeechConfig = {
-  modelsDir: string;
-  autoDownload?: boolean;
-};
 
 export type PaseoSpeechConfig = {
   dictationSttProvider?: "openai" | "local";

packages/server/src/server/speech/providers/local/config.ts

@@ -0,0 +1,176 @@
+import path from "node:path";
+
+import { z } from "zod";
+
+import type { PersistedConfig } from "../../../persisted-config.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import {
+  DEFAULT_LOCAL_STT_MODEL,
+  DEFAULT_LOCAL_TTS_MODEL,
+  LocalSttModelIdSchema,
+  LocalTtsModelIdSchema,
+  type LocalSpeechModelId,
+  type LocalSttModelId,
+  type LocalTtsModelId,
+} from "./sherpa/model-catalog.js";
+
+export type LocalSpeechProviderConfig = {
+  modelsDir: string;
+  autoDownload?: boolean;
+};
+
+export type ResolvedLocalSpeechConfig = {
+  local: LocalSpeechProviderConfig | undefined;
+  dictationLocalSttModel: LocalSttModelId;
+  voiceLocalSttModel: LocalSttModelId;
+  voiceLocalTtsModel: LocalTtsModelId;
+  voiceLocalTtsSpeakerId?: number;
+  voiceLocalTtsSpeed?: number;
+};
+
+export type { LocalSpeechModelId, LocalSttModelId, LocalTtsModelId };
+
+const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
+
+const OptionalBooleanFlagSchema = z.preprocess((value) => {
+  if (typeof value === "boolean") {
+    return value;
+  }
+  if (typeof value !== "string") {
+    return value;
+  }
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "1" || normalized === "true" || normalized === "yes") {
+    return true;
+  }
+  if (normalized === "0" || normalized === "false" || normalized === "no") {
+    return false;
+  }
+  return undefined;
+}, z.boolean().optional());
+
+const OptionalFiniteNumberSchema = z.preprocess((value) => {
+  if (typeof value === "number") {
+    return Number.isFinite(value) ? value : undefined;
+  }
+  if (typeof value !== "string") {
+    return value;
+  }
+  const parsed = Number.parseFloat(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}, z.number().optional());
+
+const OptionalIntegerSchema = z.preprocess((value) => {
+  if (typeof value === "number") {
+    return Number.isInteger(value) ? value : undefined;
+  }
+  if (typeof value !== "string") {
+    return value;
+  }
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}, z.number().int().optional());
+
+const LocalSpeechResolutionSchema = z.object({
+  includeProviderConfig: z.boolean(),
+  modelsDir: z.string().trim().min(1),
+  autoDownload: OptionalBooleanFlagSchema.default(true),
+  dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
+  voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
+  voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL),
+  voiceLocalTtsSpeakerId: OptionalIntegerSchema,
+  voiceLocalTtsSpeed: OptionalFiniteNumberSchema,
+});
+
+function persistedLocalFeatureModel(
+  provider: RequestedSpeechProviders[keyof RequestedSpeechProviders],
+  model: string | undefined
+): string | undefined {
+  if (provider !== "local") {
+    return undefined;
+  }
+  return model;
+}
+
+function shouldIncludeLocalProviderConfig(params: {
+  providers: RequestedSpeechProviders;
+  env: NodeJS.ProcessEnv;
+  persisted: PersistedConfig;
+}): boolean {
+  const localRequestedByFeature =
+    params.providers.dictationSttProvider === "local" ||
+    params.providers.voiceSttProvider === "local" ||
+    params.providers.voiceTtsProvider === "local";
+
+  return (
+    localRequestedByFeature ||
+    params.env.PASEO_LOCAL_MODELS_DIR !== undefined ||
+    params.persisted.providers?.local !== undefined
+  );
+}
+
+export function resolveLocalSpeechConfig(params: {
+  paseoHome: string;
+  env: NodeJS.ProcessEnv;
+  persisted: PersistedConfig;
+  providers: RequestedSpeechProviders;
+}): ResolvedLocalSpeechConfig {
+  const includeProviderConfig = shouldIncludeLocalProviderConfig(params);
+
+  const parsed = LocalSpeechResolutionSchema.parse({
+    includeProviderConfig,
+    modelsDir:
+      params.env.PASEO_LOCAL_MODELS_DIR ??
+      params.persisted.providers?.local?.modelsDir ??
+      path.join(params.paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR),
+    autoDownload:
+      params.env.PASEO_LOCAL_AUTO_DOWNLOAD ??
+      params.persisted.providers?.local?.autoDownload,
+    dictationLocalSttModel:
+      params.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
+      persistedLocalFeatureModel(
+        params.providers.dictationSttProvider,
+        params.persisted.features?.dictation?.stt?.model
+      ) ??
+      DEFAULT_LOCAL_STT_MODEL,
+    voiceLocalSttModel:
+      params.env.PASEO_VOICE_LOCAL_STT_MODEL ??
+      persistedLocalFeatureModel(
+        params.providers.voiceSttProvider,
+        params.persisted.features?.voiceMode?.stt?.model
+      ) ??
+      DEFAULT_LOCAL_STT_MODEL,
+    voiceLocalTtsModel:
+      params.env.PASEO_VOICE_LOCAL_TTS_MODEL ??
+      persistedLocalFeatureModel(
+        params.providers.voiceTtsProvider,
+        params.persisted.features?.voiceMode?.tts?.model
+      ) ??
+      DEFAULT_LOCAL_TTS_MODEL,
+    voiceLocalTtsSpeakerId:
+      params.env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID ??
+      params.persisted.features?.voiceMode?.tts?.speakerId,
+    voiceLocalTtsSpeed:
+      params.env.PASEO_VOICE_LOCAL_TTS_SPEED ??
+      params.persisted.features?.voiceMode?.tts?.speed,
+  });
+
+  return {
+    local:
+      parsed.includeProviderConfig
+        ? {
+            modelsDir: parsed.modelsDir,
+            autoDownload: parsed.autoDownload,
+          }
+        : undefined,
+    dictationLocalSttModel: parsed.dictationLocalSttModel,
+    voiceLocalSttModel: parsed.voiceLocalSttModel,
+    voiceLocalTtsModel: parsed.voiceLocalTtsModel,
+    ...(parsed.voiceLocalTtsSpeakerId !== undefined
+      ? { voiceLocalTtsSpeakerId: parsed.voiceLocalTtsSpeakerId }
+      : {}),
+    ...(parsed.voiceLocalTtsSpeed !== undefined
+      ? { voiceLocalTtsSpeed: parsed.voiceLocalTtsSpeed }
+      : {}),
+  };
+}

packages/server/src/server/speech/providers/local/runtime.ts

@@ -0,0 +1,372 @@
+import type { Logger } from "pino";
+
+import type { PaseoSpeechConfig } from "../../../bootstrap.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js";
+import {
+  ensureSherpaOnnxModels,
+  getSherpaOnnxModelDir,
+} from "./sherpa/model-downloader.js";
+import {
+  DEFAULT_LOCAL_STT_MODEL,
+  DEFAULT_LOCAL_TTS_MODEL,
+  LocalSttModelIdSchema,
+  LocalTtsModelIdSchema,
+  type LocalSpeechModelId,
+  type LocalSttModelId,
+  type LocalTtsModelId,
+} from "./sherpa/model-catalog.js";
+import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
+import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js";
+import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
+import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
+import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js";
+import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js";
+import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
+
+type LocalSttEngine =
+  | { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
+  | { kind: "online"; engine: SherpaOnlineRecognizerEngine };
+
+type ResolvedLocalModels = {
+  dictationLocalSttModel: LocalSttModelId;
+  voiceLocalSttModel: LocalSttModelId;
+  voiceLocalTtsModel: LocalTtsModelId;
+};
+
+type LocalSpeechAvailability = {
+  configured: boolean;
+  modelsDir: string | null;
+  autoDownload: boolean | null;
+};
+
+export type InitializedLocalSpeech = {
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+  dictationSttService: SpeechToTextProvider | null;
+  localVoiceTtsProvider: TextToSpeechProvider | null;
+  localModelConfig: {
+    modelsDir: string;
+    defaultModelIds: LocalSpeechModelId[];
+  } | null;
+  availability: LocalSpeechAvailability;
+  cleanup: () => void;
+};
+
+function buildModelDownloadHint(modelId: LocalSpeechModelId): string {
+  return `Use 'paseo speech download --model ${modelId}' to download this model.`;
+}
+
+function resolveConfiguredLocalModels(
+  speechConfig: PaseoSpeechConfig | null
+): ResolvedLocalModels {
+  return {
+    dictationLocalSttModel: LocalSttModelIdSchema.parse(
+      speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
+    ),
+    voiceLocalSttModel: LocalSttModelIdSchema.parse(
+      speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
+    ),
+    voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
+      speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL
+    ),
+  };
+}
+
+export function getLocalSpeechAvailability(
+  speechConfig: PaseoSpeechConfig | null
+): LocalSpeechAvailability {
+  const localConfig = speechConfig?.local ?? null;
+  return {
+    configured: Boolean(localConfig),
+    modelsDir: localConfig?.modelsDir ?? null,
+    autoDownload: localConfig?.autoDownload ?? null,
+  };
+}
+
+function computeRequiredLocalModelIds(params: {
+  providers: RequestedSpeechProviders;
+  models: ResolvedLocalModels;
+}): LocalSpeechModelId[] {
+  const ids = new Set<LocalSpeechModelId>();
+  if (params.providers.dictationSttProvider === "local") {
+    ids.add(params.models.dictationLocalSttModel);
+  }
+  if (params.providers.voiceSttProvider === "local") {
+    ids.add(params.models.voiceLocalSttModel);
+  }
+  if (params.providers.voiceTtsProvider === "local") {
+    ids.add(params.models.voiceLocalTtsModel);
+  }
+  return Array.from(ids);
+}
+
+async function createLocalSttEngine(params: {
+  modelId: LocalSttModelId;
+  modelsDir: string;
+  logger: Logger;
+}): Promise<LocalSttEngine> {
+  const { modelId, modelsDir, logger } = params;
+
+  if (modelId === "parakeet-tdt-0.6b-v3-int8") {
+    const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
+    return {
+      kind: "offline",
+      engine: new SherpaOfflineRecognizerEngine(
+        {
+          model: {
+            kind: "nemo_transducer",
+            encoder: `${modelDir}/encoder.int8.onnx`,
+            decoder: `${modelDir}/decoder.int8.onnx`,
+            joiner: `${modelDir}/joiner.int8.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+          },
+          numThreads: 2,
+          debug: 0,
+        },
+        logger
+      ),
+    };
+  }
+
+  if (modelId === "paraformer-bilingual-zh-en") {
+    const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
+    return {
+      kind: "online",
+      engine: new SherpaOnlineRecognizerEngine(
+        {
+          model: {
+            kind: "paraformer",
+            encoder: `${modelDir}/encoder.int8.onnx`,
+            decoder: `${modelDir}/decoder.int8.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+          },
+          numThreads: 1,
+          debug: 0,
+        },
+        logger
+      ),
+    };
+  }
+
+  if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
+    const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
+    return {
+      kind: "online",
+      engine: new SherpaOnlineRecognizerEngine(
+        {
+          model: {
+            kind: "transducer",
+            encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
+            decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
+            joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+            modelType: "zipformer",
+          },
+          numThreads: 1,
+          debug: 0,
+        },
+        logger
+      ),
+    };
+  }
+
+  throw new Error(`Unsupported local STT model '${modelId}'`);
+}
+
+export async function initializeLocalSpeechServices(params: {
+  providers: RequestedSpeechProviders;
+  speechConfig: PaseoSpeechConfig | null;
+  logger: Logger;
+}): Promise<InitializedLocalSpeech> {
+  const { providers, logger, speechConfig } = params;
+  const localConfig = speechConfig?.local ?? null;
+  const localModels = resolveConfiguredLocalModels(speechConfig);
+
+  let sttService: SpeechToTextProvider | null = null;
+  let ttsService: TextToSpeechProvider | null = null;
+  let dictationSttService: SpeechToTextProvider | null = null;
+  let localVoiceTtsProvider: TextToSpeechProvider | null = null;
+
+  const requiredLocalModelIds = computeRequiredLocalModelIds({
+    providers,
+    models: localModels,
+  });
+
+  if (requiredLocalModelIds.length > 0 && localConfig) {
+    try {
+      logger.info(
+        {
+          modelsDir: localConfig.modelsDir,
+          modelIds: requiredLocalModelIds,
+          autoDownload: localConfig.autoDownload ?? true,
+        },
+        "Ensuring local speech models"
+      );
+      await ensureSherpaOnnxModels({
+        modelsDir: localConfig.modelsDir,
+        modelIds: requiredLocalModelIds,
+        autoDownload: localConfig.autoDownload ?? true,
+        logger,
+      });
+    } catch (err) {
+      logger.error(
+        {
+          err,
+          modelsDir: localConfig.modelsDir,
+          modelIds: requiredLocalModelIds,
+          autoDownload: localConfig.autoDownload ?? true,
+          hint:
+            "Use `paseo speech models` to inspect status and " +
+            "`paseo speech download --model <MODEL_ID>` to fetch missing models.",
+        },
+        "Failed to ensure local speech models"
+      );
+    }
+  }
+
+  const localSttEngines = new Map<LocalSttModelId, LocalSttEngine>();
+
+  const getLocalSttEngine = async (
+    modelId: LocalSttModelId
+  ): Promise<LocalSttEngine | null> => {
+    const existing = localSttEngines.get(modelId);
+    if (existing) {
+      return existing;
+    }
+    if (!localConfig) {
+      return null;
+    }
+    try {
+      const created = await createLocalSttEngine({
+        modelId,
+        modelsDir: localConfig.modelsDir,
+        logger,
+      });
+      localSttEngines.set(modelId, created);
+      return created;
+    } catch (err) {
+      logger.error(
+        {
+          err,
+          modelsDir: localConfig.modelsDir,
+          modelId,
+          hint: buildModelDownloadHint(modelId),
+        },
+        "Failed to initialize local STT engine (models missing or invalid)"
+      );
+      return null;
+    }
+  };
+
+  if (providers.voiceSttProvider === "local") {
+    if (!localConfig) {
+      logger.warn(
+        { configured: false },
+        "Local STT selected for voice but local provider config is missing; STT will be unavailable"
+      );
+    } else {
+      const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel);
+      if (voiceEngine?.kind === "offline") {
+        sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
+      } else if (voiceEngine?.kind === "online") {
+        sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
+      }
+    }
+  }
+
+  if (providers.dictationSttProvider === "local") {
+    if (!localConfig) {
+      logger.warn(
+        { configured: false },
+        "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable"
+      );
+    } else {
+      const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel);
+      if (dictationEngine?.kind === "offline") {
+        dictationSttService = {
+          id: "local",
+          createSession: () =>
+            new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
+        };
+      } else if (dictationEngine?.kind === "online") {
+        dictationSttService = {
+          id: "local",
+          createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
+        };
+      }
+    }
+  }
+
+  if (providers.voiceTtsProvider === "local") {
+    if (!localConfig) {
+      logger.warn(
+        { configured: false },
+        "Local TTS selected for voice but local provider config is missing; TTS will be unavailable"
+      );
+    } else {
+      try {
+        if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
+          const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
+          localVoiceTtsProvider = await PocketTtsOnnxTTS.create(
+            {
+              modelDir,
+              precision: "int8",
+              targetChunkMs: 50,
+            },
+            logger
+          );
+        } else {
+          const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
+          localVoiceTtsProvider = new SherpaOnnxTTS(
+            {
+              preset: localModels.voiceLocalTtsModel,
+              modelDir,
+              speakerId: speechConfig?.voiceLocalTtsSpeakerId,
+              speed: speechConfig?.voiceLocalTtsSpeed,
+            },
+            logger
+          );
+        }
+        ttsService = localVoiceTtsProvider;
+      } catch (err) {
+        logger.error(
+          {
+            err,
+            modelsDir: localConfig.modelsDir,
+            modelId: localModels.voiceLocalTtsModel,
+            hint: buildModelDownloadHint(localModels.voiceLocalTtsModel),
+          },
+          "Failed to initialize local TTS engine (models missing or invalid)"
+        );
+      }
+    }
+  }
+
+  const cleanup = () => {
+    const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null;
+    if (typeof maybeFreeable?.free === "function") {
+      maybeFreeable.free();
+    }
+    for (const engine of localSttEngines.values()) {
+      engine.engine.free();
+    }
+  };
+
+  return {
+    sttService,
+    ttsService,
+    dictationSttService,
+    localVoiceTtsProvider,
+    localModelConfig:
+      localConfig
+        ? {
+            modelsDir: localConfig.modelsDir,
+            defaultModelIds: requiredLocalModelIds,
+          }
+        : null,
+    availability: getLocalSpeechAvailability(speechConfig),
+    cleanup,
+  };
+}

packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts

@@ -115,6 +115,7 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
 } as const satisfies Record<string, SherpaOnnxCatalogEntry>;
 
 export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG;
+export type LocalSpeechModelId = SherpaOnnxModelId;
 
 type ModelIdByKind<K extends SherpaOnnxModelKind> = {
   [Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K

packages/server/src/server/speech/providers/openai/config.ts

@@ -0,0 +1,132 @@
+import { z } from "zod";
+
+import type { PersistedConfig } from "../../../persisted-config.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import type { STTConfig } from "./stt.js";
+import type { TTSConfig } from "./tts.js";
+
+export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
+export const DEFAULT_OPENAI_TTS_MODEL = "tts-1";
+
+export type OpenAiSpeechProviderConfig = {
+  apiKey?: string;
+  stt?: Partial<STTConfig> & { apiKey?: string };
+  tts?: Partial<TTSConfig> & { apiKey?: string };
+  realtimeTranscriptionModel?: string;
+};
+
+const OpenAiTtsVoiceSchema = z.enum([
+  "alloy",
+  "echo",
+  "fable",
+  "onyx",
+  "nova",
+  "shimmer",
+]);
+
+const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
+
+const OptionalFiniteNumberSchema = z.preprocess((value) => {
+  if (typeof value === "number") {
+    return Number.isFinite(value) ? value : undefined;
+  }
+  if (typeof value !== "string") {
+    return value;
+  }
+  const parsed = Number.parseFloat(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}, z.number().optional());
+
+const OptionalTrimmedStringSchema = z.preprocess((value) => {
+  if (typeof value !== "string") {
+    return value;
+  }
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : undefined;
+}, z.string().optional());
+
+const OpenAiSpeechResolutionSchema = z.object({
+  apiKey: OptionalTrimmedStringSchema,
+  sttConfidenceThreshold: OptionalFiniteNumberSchema,
+  sttModel: OptionalTrimmedStringSchema,
+  ttsVoice: z.preprocess((value) => {
+    if (typeof value !== "string") {
+      return value;
+    }
+    const normalized = value.trim().toLowerCase();
+    return normalized.length > 0 ? normalized : undefined;
+  }, OpenAiTtsVoiceSchema.default("alloy")),
+  ttsModel: z.preprocess((value) => {
+    if (typeof value !== "string") {
+      return value;
+    }
+    const normalized = value.trim().toLowerCase();
+    return normalized.length > 0 ? normalized : undefined;
+  }, OpenAiTtsModelSchema.default(DEFAULT_OPENAI_TTS_MODEL)),
+  realtimeTranscriptionModel: OptionalTrimmedStringSchema.default(
+    DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL
+  ),
+});
+
+export function resolveOpenAiSpeechConfig(params: {
+  env: NodeJS.ProcessEnv;
+  persisted: PersistedConfig;
+  providers: RequestedSpeechProviders;
+}): OpenAiSpeechProviderConfig | undefined {
+  const parsed = OpenAiSpeechResolutionSchema.parse({
+    apiKey: params.env.OPENAI_API_KEY ?? params.persisted.providers?.openai?.apiKey,
+    sttConfidenceThreshold:
+      params.env.STT_CONFIDENCE_THRESHOLD ??
+      params.persisted.features?.dictation?.stt?.confidenceThreshold,
+    sttModel:
+      params.env.STT_MODEL ??
+      (params.providers.voiceSttProvider === "openai"
+        ? params.persisted.features?.voiceMode?.stt?.model
+        : undefined) ??
+      (params.providers.dictationSttProvider === "openai"
+        ? params.persisted.features?.dictation?.stt?.model
+        : undefined),
+    ttsVoice:
+      params.env.TTS_VOICE ??
+      (params.providers.voiceTtsProvider === "openai"
+        ? params.persisted.features?.voiceMode?.tts?.voice
+        : undefined) ??
+      "alloy",
+    ttsModel:
+      params.env.TTS_MODEL ??
+      (params.providers.voiceTtsProvider === "openai"
+        ? params.persisted.features?.voiceMode?.tts?.model
+        : undefined) ??
+      DEFAULT_OPENAI_TTS_MODEL,
+    realtimeTranscriptionModel:
+      params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ??
+      (params.providers.dictationSttProvider === "openai"
+        ? params.persisted.features?.dictation?.stt?.model
+        : undefined) ??
+      DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+  });
+
+  if (!parsed.apiKey) {
+    return undefined;
+  }
+
+  return {
+    apiKey: parsed.apiKey,
+    stt: {
+      apiKey: parsed.apiKey,
+      ...(parsed.sttConfidenceThreshold !== undefined
+        ? { confidenceThreshold: parsed.sttConfidenceThreshold }
+        : {}),
+      ...(parsed.sttModel
+        ? { model: parsed.sttModel }
+        : {}),
+    },
+    tts: {
+      apiKey: parsed.apiKey,
+      voice: parsed.ttsVoice,
+      model: parsed.ttsModel,
+      responseFormat: "pcm",
+    },
+    realtimeTranscriptionModel: parsed.realtimeTranscriptionModel,
+  };
+}

packages/server/src/server/speech/providers/openai/runtime.ts

@@ -0,0 +1,167 @@
+import type { Logger } from "pino";
+
+import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import {
+  DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+  DEFAULT_OPENAI_TTS_MODEL,
+  type OpenAiSpeechProviderConfig,
+} from "./config.js";
+import { OpenAIRealtimeTranscriptionSession } from "./realtime-transcription-session.js";
+import { OpenAISTT } from "./stt.js";
+import { OpenAITTS } from "./tts.js";
+
+type OpenAiCredentialState = {
+  openaiSttApiKey: string | undefined;
+  openaiTtsApiKey: string | undefined;
+  openaiDictationApiKey: string | undefined;
+};
+
+export type OpenAiSpeechAvailability = {
+  stt: boolean;
+  tts: boolean;
+  dictationStt: boolean;
+};
+
+export type SpeechServices = {
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+  dictationSttService: SpeechToTextProvider | null;
+};
+
+function resolveOpenAiCredentials(
+  openaiConfig: OpenAiSpeechProviderConfig | undefined
+): OpenAiCredentialState {
+  const openaiApiKey = openaiConfig?.apiKey;
+  return {
+    openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey,
+    openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey,
+    openaiDictationApiKey: openaiApiKey,
+  };
+}
+
+export function getOpenAiSpeechAvailability(
+  openaiConfig: OpenAiSpeechProviderConfig | undefined
+): OpenAiSpeechAvailability {
+  const credentials = resolveOpenAiCredentials(openaiConfig);
+  return {
+    stt: Boolean(credentials.openaiSttApiKey),
+    tts: Boolean(credentials.openaiTtsApiKey),
+    dictationStt: Boolean(credentials.openaiDictationApiKey),
+  };
+}
+
+export function validateOpenAiCredentialRequirements(params: {
+  providers: RequestedSpeechProviders;
+  openaiConfig: OpenAiSpeechProviderConfig | undefined;
+  logger: Logger;
+}): void {
+  const { providers, logger, openaiConfig } = params;
+  const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
+
+  const missingOpenAiCredentialsFor: string[] = [];
+  if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) {
+    missingOpenAiCredentialsFor.push("voice.stt");
+  }
+  if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) {
+    missingOpenAiCredentialsFor.push("voice.tts");
+  }
+  if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) {
+    missingOpenAiCredentialsFor.push("dictation.stt");
+  }
+
+  if (missingOpenAiCredentialsFor.length > 0) {
+    logger.error(
+      {
+        requestedProviders: {
+          dictationStt: providers.dictationSttProvider,
+          voiceStt: providers.voiceSttProvider,
+          voiceTts: providers.voiceTtsProvider,
+        },
+        missingOpenAiCredentialsFor,
+      },
+      "Invalid speech configuration: OpenAI provider selected but credentials are missing"
+    );
+    throw new Error(
+      `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
+    );
+  }
+}
+
+export function initializeOpenAiSpeechServices(params: {
+  providers: RequestedSpeechProviders;
+  openaiConfig: OpenAiSpeechProviderConfig | undefined;
+  existing: SpeechServices;
+  logger: Logger;
+}): SpeechServices {
+  const { providers, openaiConfig, existing, logger } = params;
+  const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
+
+  let sttService = existing.sttService;
+  let ttsService = existing.ttsService;
+  let dictationSttService = existing.dictationSttService;
+
+  const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai";
+  const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai";
+  const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai";
+
+  if (
+    (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
+    (openAiCredentials.openaiSttApiKey ||
+      openAiCredentials.openaiTtsApiKey ||
+      openAiCredentials.openaiDictationApiKey)
+  ) {
+    logger.info("OpenAI speech provider initialized");
+
+    if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) {
+      const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {};
+      sttService = new OpenAISTT(
+        {
+          apiKey: openAiCredentials.openaiSttApiKey,
+          ...sttConfig,
+        },
+        logger
+      );
+    }
+
+    if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) {
+      const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {};
+      ttsService = new OpenAITTS(
+        {
+          apiKey: openAiCredentials.openaiTtsApiKey,
+          voice: "alloy",
+          model: DEFAULT_OPENAI_TTS_MODEL,
+          responseFormat: "pcm",
+          ...ttsConfig,
+        },
+        logger
+      );
+    }
+
+    const dictationApiKey = openAiCredentials.openaiDictationApiKey;
+    if (needsOpenAiDictation && dictationApiKey) {
+      dictationSttService = {
+        id: "openai",
+        createSession: ({ logger: sessionLogger, language, prompt }) =>
+          new OpenAIRealtimeTranscriptionSession({
+            apiKey: dictationApiKey,
+            logger: sessionLogger,
+            transcriptionModel:
+              openaiConfig?.realtimeTranscriptionModel ??
+              DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+            ...(language ? { language } : {}),
+            ...(prompt ? { prompt } : {}),
+            turnDetection: null,
+          }),
+      };
+    }
+  } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
+    logger.warn("OpenAI speech providers are configured but credentials are missing");
+  }
+
+  return {
+    sttService,
+    ttsService,
+    dictationSttService,
+  };
+}

packages/server/src/server/speech/speech-config-resolver.ts

@@ -1,33 +1,13 @@
-import path from "node:path";
-
 import { z } from "zod";
 
 import type { PersistedConfig } from "../persisted-config.js";
 import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
+import { resolveLocalSpeechConfig } from "./providers/local/config.js";
+import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js";
 import {
-  LocalSttModelIdSchema,
+  SpeechProviderIdSchema,
-  LocalTtsModelIdSchema,
+  type RequestedSpeechProviders,
-} from "./providers/local/sherpa/model-catalog.js";
+} from "./speech-types.js";
-import {
-  DEFAULT_LOCAL_STT_MODEL,
-  DEFAULT_LOCAL_TTS_MODEL,
-  DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
-  DEFAULT_OPENAI_TTS_MODEL,
-} from "./speech-defaults.js";
-import { SpeechProviderIdSchema } from "./speech-types.js";
-
-const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
-
-const OpenAiTtsVoiceSchema = z.enum([
-  "alloy",
-  "echo",
-  "fable",
-  "onyx",
-  "nova",
-  "shimmer",
-]);
-
-const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
 
 const OptionalSpeechProviderSchema = z.preprocess((value) => {
   if (typeof value !== "string") {
@@ -37,136 +17,31 @@ const OptionalSpeechProviderSchema = z.preprocess((value) => {
   return normalized.length > 0 ? normalized : undefined;
 }, SpeechProviderIdSchema.optional());
 
-const OptionalBooleanFlagSchema = z.preprocess((value) => {
+const RequestedSpeechProvidersSchema = z.object({
-  if (typeof value === "boolean") {
+  dictationSttProvider: OptionalSpeechProviderSchema.default("local"),
-    return value;
+  voiceSttProvider: OptionalSpeechProviderSchema.default("local"),
-  }
+  voiceTtsProvider: OptionalSpeechProviderSchema.default("local"),
-  if (typeof value !== "string") {
+});
-    return value;
-  }
-  const normalized = value.trim().toLowerCase();
-  if (normalized === "1" || normalized === "true" || normalized === "yes") {
-    return true;
-  }
-  if (normalized === "0" || normalized === "false" || normalized === "no") {
-    return false;
-  }
-  return undefined;
-}, z.boolean().optional());
 
-const OptionalFiniteNumberSchema = z.preprocess((value) => {
+function resolveRequestedSpeechProviders(params: {
-  if (typeof value === "number") {
+  env: NodeJS.ProcessEnv;
-    return Number.isFinite(value) ? value : undefined;
+  persisted: PersistedConfig;
-  }
+}): RequestedSpeechProviders {
-  if (typeof value !== "string") {
+  return RequestedSpeechProvidersSchema.parse({
-    return value;
+    dictationSttProvider:
-  }
+      params.env.PASEO_DICTATION_STT_PROVIDER ??
-  const parsed = Number.parseFloat(value);
+      params.persisted.features?.dictation?.stt?.provider ??
-  return Number.isFinite(parsed) ? parsed : undefined;
+      "local",
-}, z.number().optional());
+    voiceSttProvider:
-
+      params.env.PASEO_VOICE_STT_PROVIDER ??
-const OptionalIntegerSchema = z.preprocess((value) => {
+      params.persisted.features?.voiceMode?.stt?.provider ??
-  if (typeof value === "number") {
+      "local",
-    return Number.isInteger(value) ? value : undefined;
+    voiceTtsProvider:
-  }
+      params.env.PASEO_VOICE_TTS_PROVIDER ??
-  if (typeof value !== "string") {
+      params.persisted.features?.voiceMode?.tts?.provider ??
-    return value;
+      "local",
-  }
-  const parsed = Number.parseInt(value, 10);
-  return Number.isFinite(parsed) ? parsed : undefined;
-}, z.number().int().optional());
-
-const OptionalTrimmedStringSchema = z.preprocess((value) => {
-  if (typeof value !== "string") {
-    return value;
-  }
-  const trimmed = value.trim();
-  return trimmed.length > 0 ? trimmed : undefined;
-}, z.string().optional());
-
-const ResolvedSpeechConfigSchema = z
-  .object({
-    dictationSttProvider: OptionalSpeechProviderSchema.default("local"),
-    voiceSttProvider: OptionalSpeechProviderSchema.default("local"),
-    voiceTtsProvider: OptionalSpeechProviderSchema.default("local"),
-    localModelsDir: z.string().trim().min(1),
-    localAutoDownload: OptionalBooleanFlagSchema.default(true),
-    dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
-    voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
-    voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL),
-    voiceLocalTtsSpeakerId: OptionalIntegerSchema,
-    voiceLocalTtsSpeed: OptionalFiniteNumberSchema,
-    anyLocalRequested: z.boolean(),
-    openaiApiKey: OptionalTrimmedStringSchema,
-    openaiSttConfidenceThreshold: OptionalFiniteNumberSchema,
-    openaiSttModel: OptionalTrimmedStringSchema,
-    openaiTtsVoice: z.preprocess((value) => {
-      if (typeof value !== "string") {
-        return value;
-      }
-      const normalized = value.trim().toLowerCase();
-      return normalized.length > 0 ? normalized : undefined;
-    }, OpenAiTtsVoiceSchema.default("alloy")),
-    openaiTtsModel: z.preprocess((value) => {
-      if (typeof value !== "string") {
-        return value;
-      }
-      const normalized = value.trim().toLowerCase();
-      return normalized.length > 0 ? normalized : undefined;
-    }, OpenAiTtsModelSchema.default(DEFAULT_OPENAI_TTS_MODEL)),
-    openaiRealtimeTranscriptionModel: OptionalTrimmedStringSchema.default(
-      DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL
-    ),
-  })
-  .transform((input): { openai: PaseoOpenAIConfig | undefined; speech: PaseoSpeechConfig } => {
-    const openai = input.openaiApiKey
-      ? {
-          apiKey: input.openaiApiKey,
-          stt: {
-            apiKey: input.openaiApiKey,
-            ...(input.openaiSttConfidenceThreshold !== undefined
-              ? { confidenceThreshold: input.openaiSttConfidenceThreshold }
-              : {}),
-            ...(input.openaiSttModel
-              ? { model: input.openaiSttModel }
-              : {}),
-          },
-          tts: {
-            apiKey: input.openaiApiKey,
-            voice: input.openaiTtsVoice,
-            model: input.openaiTtsModel,
-            responseFormat: "pcm" as const,
-          },
-          realtimeTranscriptionModel: input.openaiRealtimeTranscriptionModel,
-        }
-      : undefined;
-
-    return {
-      openai,
-      speech: {
-        dictationSttProvider: input.dictationSttProvider,
-        voiceSttProvider: input.voiceSttProvider,
-        voiceTtsProvider: input.voiceTtsProvider,
-        ...(input.anyLocalRequested
-          ? {
-              local: {
-                modelsDir: input.localModelsDir,
-                autoDownload: input.localAutoDownload,
-              },
-            }
-          : {}),
-        dictationLocalSttModel: input.dictationLocalSttModel,
-        voiceLocalSttModel: input.voiceLocalSttModel,
-        voiceLocalTtsModel: input.voiceLocalTtsModel,
-        ...(input.voiceLocalTtsSpeakerId !== undefined
-          ? { voiceLocalTtsSpeakerId: input.voiceLocalTtsSpeakerId }
-          : {}),
-        ...(input.voiceLocalTtsSpeed !== undefined
-          ? { voiceLocalTtsSpeed: input.voiceLocalTtsSpeed }
-          : {}),
-      },
-    };
   });
+}
 
 export function resolveSpeechConfig(params: {
   paseoHome: string;
@@ -176,89 +51,42 @@ export function resolveSpeechConfig(params: {
   openai: PaseoOpenAIConfig | undefined;
   speech: PaseoSpeechConfig;
 } {
-  const { paseoHome, env, persisted } = params;
+  const providers = resolveRequestedSpeechProviders({
-
+    env: params.env,
-  const dictationSttProvider =
+    persisted: params.persisted,
-    env.PASEO_DICTATION_STT_PROVIDER
-    ?? persisted.features?.dictation?.stt?.provider
-    ?? "local";
-
-  const voiceSttProvider =
-    env.PASEO_VOICE_STT_PROVIDER
-    ?? persisted.features?.voiceMode?.stt?.provider
-    ?? "local";
-
-  const voiceTtsProvider =
-    env.PASEO_VOICE_TTS_PROVIDER
-    ?? persisted.features?.voiceMode?.tts?.provider
-    ?? "local";
-
-  const anyLocalRequested =
-    dictationSttProvider === "local" ||
-    voiceSttProvider === "local" ||
-    voiceTtsProvider === "local" ||
-    env.PASEO_LOCAL_MODELS_DIR !== undefined ||
-    persisted.providers?.local !== undefined;
-
-  return ResolvedSpeechConfigSchema.parse({
-    dictationSttProvider,
-    voiceSttProvider,
-    voiceTtsProvider,
-    localModelsDir:
-      env.PASEO_LOCAL_MODELS_DIR
-      ?? persisted.providers?.local?.modelsDir
-      ?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR),
-    localAutoDownload:
-      env.PASEO_LOCAL_AUTO_DOWNLOAD
-      ?? persisted.providers?.local?.autoDownload,
-    dictationLocalSttModel:
-      env.PASEO_DICTATION_LOCAL_STT_MODEL
-      ?? persisted.features?.dictation?.stt?.model
-      ?? DEFAULT_LOCAL_STT_MODEL,
-    voiceLocalSttModel:
-      env.PASEO_VOICE_LOCAL_STT_MODEL
-      ?? persisted.features?.voiceMode?.stt?.model
-      ?? DEFAULT_LOCAL_STT_MODEL,
-    voiceLocalTtsModel:
-      env.PASEO_VOICE_LOCAL_TTS_MODEL
-      ?? persisted.features?.voiceMode?.tts?.model
-      ?? DEFAULT_LOCAL_TTS_MODEL,
-    voiceLocalTtsSpeakerId:
-      env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID
-      ?? persisted.features?.voiceMode?.tts?.speakerId,
-    voiceLocalTtsSpeed:
-      env.PASEO_VOICE_LOCAL_TTS_SPEED
-      ?? persisted.features?.voiceMode?.tts?.speed,
-    anyLocalRequested,
-    openaiApiKey: env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey,
-    openaiSttConfidenceThreshold:
-      env.STT_CONFIDENCE_THRESHOLD
-      ?? persisted.features?.dictation?.stt?.confidenceThreshold,
-    openaiSttModel:
-      env.STT_MODEL
-      ?? (voiceSttProvider === "openai"
-        ? persisted.features?.voiceMode?.stt?.model
-        : undefined)
-      ?? (dictationSttProvider === "openai"
-        ? persisted.features?.dictation?.stt?.model
-        : undefined),
-    openaiTtsVoice:
-      env.TTS_VOICE
-      ?? (voiceTtsProvider === "openai"
-        ? persisted.features?.voiceMode?.tts?.voice
-        : undefined)
-      ?? "alloy",
-    openaiTtsModel:
-      env.TTS_MODEL
-      ?? (voiceTtsProvider === "openai"
-        ? persisted.features?.voiceMode?.tts?.model
-        : undefined)
-      ?? DEFAULT_OPENAI_TTS_MODEL,
-    openaiRealtimeTranscriptionModel:
-      env.OPENAI_REALTIME_TRANSCRIPTION_MODEL
-      ?? (dictationSttProvider === "openai"
-        ? persisted.features?.dictation?.stt?.model
-        : undefined)
-      ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
   });
+
+  const local = resolveLocalSpeechConfig({
+    paseoHome: params.paseoHome,
+    env: params.env,
+    persisted: params.persisted,
+    providers,
+  });
+
+  const openai = resolveOpenAiSpeechConfig({
+    env: params.env,
+    persisted: params.persisted,
+    providers,
+  });
+
+  return {
+    openai,
+    speech: {
+      dictationSttProvider: providers.dictationSttProvider,
+      voiceSttProvider: providers.voiceSttProvider,
+      voiceTtsProvider: providers.voiceTtsProvider,
+      ...(local.local
+        ? { local: local.local }
+        : {}),
+      dictationLocalSttModel: local.dictationLocalSttModel,
+      voiceLocalSttModel: local.voiceLocalSttModel,
+      voiceLocalTtsModel: local.voiceLocalTtsModel,
+      ...(local.voiceLocalTtsSpeakerId !== undefined
+        ? { voiceLocalTtsSpeakerId: local.voiceLocalTtsSpeakerId }
+        : {}),
+      ...(local.voiceLocalTtsSpeed !== undefined
+        ? { voiceLocalTtsSpeed: local.voiceLocalTtsSpeed }
+        : {}),
+    },
+  };
 }

packages/server/src/server/speech/speech-defaults.ts

@@ -1,9 +0,0 @@
-import {
-  DEFAULT_LOCAL_STT_MODEL,
-  DEFAULT_LOCAL_TTS_MODEL,
-} from "./providers/local/sherpa/model-catalog.js";
-
-export { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL };
-
-export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
-export const DEFAULT_OPENAI_TTS_MODEL = "tts-1";

packages/server/src/server/speech/speech-runtime.ts

@@ -2,71 +2,17 @@ import type { Logger } from "pino";
 
 import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
 import {
-  OpenAISTT,
+  getLocalSpeechAvailability,
-} from "./providers/openai/stt.js";
+  initializeLocalSpeechServices,
+} from "./providers/local/runtime.js";
+import type { LocalSpeechModelId } from "./providers/local/config.js";
 import {
-  OpenAITTS,
+  getOpenAiSpeechAvailability,
-} from "./providers/openai/tts.js";
+  initializeOpenAiSpeechServices,
-import { OpenAIRealtimeTranscriptionSession } from "./providers/openai/realtime-transcription-session.js";
+  validateOpenAiCredentialRequirements,
+} from "./providers/openai/runtime.js";
 import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
-import { SherpaOnlineRecognizerEngine } from "./providers/local/sherpa/sherpa-online-recognizer.js";
+import type { RequestedSpeechProviders } from "./speech-types.js";
-import { SherpaOfflineRecognizerEngine } from "./providers/local/sherpa/sherpa-offline-recognizer.js";
-import { SherpaOnnxSTT } from "./providers/local/sherpa/sherpa-stt.js";
-import { SherpaOnnxParakeetSTT } from "./providers/local/sherpa/sherpa-parakeet-stt.js";
-import { SherpaOnnxTTS } from "./providers/local/sherpa/sherpa-tts.js";
-import { SherpaRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-realtime-session.js";
-import { SherpaParakeetRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-parakeet-realtime-session.js";
-import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./providers/local/sherpa/model-downloader.js";
-import {
-  LocalSttModelIdSchema,
-  LocalTtsModelIdSchema,
-  type LocalSttModelId,
-  type LocalTtsModelId,
-  type SherpaOnnxModelId,
-} from "./providers/local/sherpa/model-catalog.js";
-import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js";
-import {
-  DEFAULT_LOCAL_STT_MODEL,
-  DEFAULT_LOCAL_TTS_MODEL,
-  DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
-  DEFAULT_OPENAI_TTS_MODEL,
-} from "./speech-defaults.js";
-import type { SpeechProviderId } from "./speech-types.js";
-
-type LocalSttEngine =
-  | { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
-  | { kind: "online"; engine: SherpaOnlineRecognizerEngine };
-
-type RequestedSpeechProviders = {
-  dictationSttProvider: SpeechProviderId;
-  voiceSttProvider: SpeechProviderId;
-  voiceTtsProvider: SpeechProviderId;
-};
-
-type ResolvedLocalModels = {
-  dictationLocalSttModel: LocalSttModelId;
-  voiceLocalSttModel: LocalSttModelId;
-  voiceLocalTtsModel: LocalTtsModelId;
-};
-
-type OpenAiCredentialState = {
-  openaiSttApiKey: string | undefined;
-  openaiTtsApiKey: string | undefined;
-  openaiDictationApiKey: string | undefined;
-};
-
-type InitializedLocalSpeech = {
-  sttService: SpeechToTextProvider | null;
-  ttsService: TextToSpeechProvider | null;
-  dictationSttService: SpeechToTextProvider | null;
-  localVoiceTtsProvider: TextToSpeechProvider | null;
-  localSttEngines: Map<LocalSttModelId, LocalSttEngine>;
-  requiredLocalModelIds: SherpaOnnxModelId[];
-};
-
-function buildModelDownloadHint(modelId: SherpaOnnxModelId): string {
-  return `Use 'paseo speech download --model ${modelId}' to download this model.`;
-}
 
 function resolveRequestedSpeechProviders(
   speechConfig: PaseoSpeechConfig | null
@@ -78,425 +24,6 @@ function resolveRequestedSpeechProviders(
   };
 }
 
-function resolveConfiguredLocalModels(
-  speechConfig: PaseoSpeechConfig | null
-): ResolvedLocalModels {
-  return {
-    dictationLocalSttModel: LocalSttModelIdSchema.parse(
-      speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
-    ),
-    voiceLocalSttModel: LocalSttModelIdSchema.parse(
-      speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
-    ),
-    voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
-      speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL
-    ),
-  };
-}
-
-function computeRequiredLocalModelIds(params: {
-  providers: RequestedSpeechProviders;
-  models: {
-    dictationLocalSttModel: SherpaOnnxModelId;
-    voiceLocalSttModel: SherpaOnnxModelId;
-    voiceLocalTtsModel: SherpaOnnxModelId;
-  };
-}): SherpaOnnxModelId[] {
-  const ids = new Set<SherpaOnnxModelId>();
-  if (params.providers.dictationSttProvider === "local") {
-    ids.add(params.models.dictationLocalSttModel);
-  }
-  if (params.providers.voiceSttProvider === "local") {
-    ids.add(params.models.voiceLocalSttModel);
-  }
-  if (params.providers.voiceTtsProvider === "local") {
-    ids.add(params.models.voiceLocalTtsModel);
-  }
-  return Array.from(ids);
-}
-
-function resolveOpenAiCredentials(openaiConfig?: PaseoOpenAIConfig): OpenAiCredentialState {
-  const openaiApiKey = openaiConfig?.apiKey;
-  return {
-    openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey,
-    openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey,
-    openaiDictationApiKey: openaiApiKey,
-  };
-}
-
-function validateOpenAiCredentialRequirements(params: {
-  providers: RequestedSpeechProviders;
-  openAiCredentials: OpenAiCredentialState;
-  logger: Logger;
-}): void {
-  const { providers, openAiCredentials, logger } = params;
-  const missingOpenAiCredentialsFor: string[] = [];
-  if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) {
-    missingOpenAiCredentialsFor.push("voice.stt");
-  }
-  if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) {
-    missingOpenAiCredentialsFor.push("voice.tts");
-  }
-  if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) {
-    missingOpenAiCredentialsFor.push("dictation.stt");
-  }
-
-  if (missingOpenAiCredentialsFor.length > 0) {
-    logger.error(
-      {
-        requestedProviders: {
-          dictationStt: providers.dictationSttProvider,
-          voiceStt: providers.voiceSttProvider,
-          voiceTts: providers.voiceTtsProvider,
-        },
-        missingOpenAiCredentialsFor,
-      },
-      "Invalid speech configuration: OpenAI provider selected but credentials are missing"
-    );
-    throw new Error(
-      `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
-    );
-  }
-}
-
-async function createLocalSttEngine(params: {
-  modelId: LocalSttModelId;
-  modelsDir: string;
-  logger: Logger;
-}): Promise<LocalSttEngine> {
-  const { modelId, modelsDir, logger } = params;
-
-  if (modelId === "parakeet-tdt-0.6b-v3-int8") {
-    const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
-    return {
-      kind: "offline",
-      engine: new SherpaOfflineRecognizerEngine(
-        {
-          model: {
-            kind: "nemo_transducer",
-            encoder: `${modelDir}/encoder.int8.onnx`,
-            decoder: `${modelDir}/decoder.int8.onnx`,
-            joiner: `${modelDir}/joiner.int8.onnx`,
-            tokens: `${modelDir}/tokens.txt`,
-          },
-          numThreads: 2,
-          debug: 0,
-        },
-        logger
-      ),
-    };
-  }
-
-  if (modelId === "paraformer-bilingual-zh-en") {
-    const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
-    return {
-      kind: "online",
-      engine: new SherpaOnlineRecognizerEngine(
-        {
-          model: {
-            kind: "paraformer",
-            encoder: `${modelDir}/encoder.int8.onnx`,
-            decoder: `${modelDir}/decoder.int8.onnx`,
-            tokens: `${modelDir}/tokens.txt`,
-          },
-          numThreads: 1,
-          debug: 0,
-        },
-        logger
-      ),
-    };
-  }
-
-  if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
-    const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
-    return {
-      kind: "online",
-      engine: new SherpaOnlineRecognizerEngine(
-        {
-          model: {
-            kind: "transducer",
-            encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
-            decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
-            joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
-            tokens: `${modelDir}/tokens.txt`,
-            modelType: "zipformer",
-          },
-          numThreads: 1,
-          debug: 0,
-        },
-        logger
-      ),
-    };
-  }
-
-  throw new Error(`Unsupported local STT model '${modelId}'`);
-}
-
-async function initializeLocalSpeechServices(params: {
-  providers: RequestedSpeechProviders;
-  localConfig: NonNullable<PaseoSpeechConfig["local"]> | null;
-  localModels: ResolvedLocalModels;
-  speechConfig: PaseoSpeechConfig | null;
-  logger: Logger;
-}): Promise<InitializedLocalSpeech> {
-  const { providers, localConfig, localModels, speechConfig, logger } = params;
-
-  const sttServices = {
-    sttService: null as SpeechToTextProvider | null,
-    ttsService: null as TextToSpeechProvider | null,
-    dictationSttService: null as SpeechToTextProvider | null,
-  };
-
-  const localSttEngines = new Map<LocalSttModelId, LocalSttEngine>();
-  let localVoiceTtsProvider: TextToSpeechProvider | null = null;
-
-  const requiredLocalModelIds = computeRequiredLocalModelIds({
-    providers,
-    models: localModels,
-  });
-
-  if (requiredLocalModelIds.length > 0 && localConfig) {
-    try {
-      logger.info(
-        {
-          modelsDir: localConfig.modelsDir,
-          modelIds: requiredLocalModelIds,
-          autoDownload: localConfig.autoDownload ?? true,
-        },
-        "Ensuring local speech models"
-      );
-      await ensureSherpaOnnxModels({
-        modelsDir: localConfig.modelsDir,
-        modelIds: requiredLocalModelIds,
-        autoDownload: localConfig.autoDownload ?? true,
-        logger,
-      });
-    } catch (err) {
-      logger.error(
-        {
-          err,
-          modelsDir: localConfig.modelsDir,
-          modelIds: requiredLocalModelIds,
-          autoDownload: localConfig.autoDownload ?? true,
-          hint:
-            "Use `paseo speech models` to inspect status and " +
-            "`paseo speech download --model <MODEL_ID>` to fetch missing models.",
-        },
-        "Failed to ensure local speech models"
-      );
-    }
-  }
-
-  const getLocalSttEngine = async (
-    modelId: LocalSttModelId
-  ): Promise<LocalSttEngine | null> => {
-    const existing = localSttEngines.get(modelId);
-    if (existing) {
-      return existing;
-    }
-    if (!localConfig) {
-      return null;
-    }
-    try {
-      const created = await createLocalSttEngine({
-        modelId,
-        modelsDir: localConfig.modelsDir,
-        logger,
-      });
-      localSttEngines.set(modelId, created);
-      return created;
-    } catch (err) {
-      logger.error(
-        {
-          err,
-          modelsDir: localConfig.modelsDir,
-          modelId,
-          hint: buildModelDownloadHint(modelId),
-        },
-        "Failed to initialize local STT engine (models missing or invalid)"
-      );
-      return null;
-    }
-  };
-
-  if (providers.voiceSttProvider === "local") {
-    if (!localConfig) {
-      logger.warn(
-        { configured: false },
-        "Local STT selected for voice but local provider config is missing; STT will be unavailable"
-      );
-    } else {
-      const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel);
-      if (voiceEngine?.kind === "offline") {
-        sttServices.sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
-      } else if (voiceEngine?.kind === "online") {
-        sttServices.sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
-      }
-    }
-  }
-
-  if (providers.dictationSttProvider === "local") {
-    if (!localConfig) {
-      logger.warn(
-        { configured: false },
-        "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable"
-      );
-    } else {
-      const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel);
-      if (dictationEngine?.kind === "offline") {
-        sttServices.dictationSttService = {
-          id: "local",
-          createSession: () =>
-            new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
-        };
-      } else if (dictationEngine?.kind === "online") {
-        sttServices.dictationSttService = {
-          id: "local",
-          createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
-        };
-      }
-    }
-  }
-
-  if (providers.voiceTtsProvider === "local") {
-    if (!localConfig) {
-      logger.warn(
-        { configured: false },
-        "Local TTS selected for voice but local provider config is missing; TTS will be unavailable"
-      );
-    } else {
-      try {
-        if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
-          const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
-          localVoiceTtsProvider = await PocketTtsOnnxTTS.create(
-            {
-              modelDir,
-              precision: "int8",
-              targetChunkMs: 50,
-            },
-            logger
-          );
-        } else {
-          const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
-          localVoiceTtsProvider = new SherpaOnnxTTS(
-            {
-              preset: localModels.voiceLocalTtsModel,
-              modelDir,
-              speakerId: speechConfig?.voiceLocalTtsSpeakerId,
-              speed: speechConfig?.voiceLocalTtsSpeed,
-            },
-            logger
-          );
-        }
-        sttServices.ttsService = localVoiceTtsProvider;
-      } catch (err) {
-        logger.error(
-          {
-            err,
-            modelsDir: localConfig.modelsDir,
-            modelId: localModels.voiceLocalTtsModel,
-            hint: buildModelDownloadHint(localModels.voiceLocalTtsModel),
-          },
-          "Failed to initialize local TTS engine (models missing or invalid)"
-        );
-      }
-    }
-  }
-
-  return {
-    ...sttServices,
-    localVoiceTtsProvider,
-    localSttEngines,
-    requiredLocalModelIds,
-  };
-}
-
-function initializeOpenAiSpeechServices(params: {
-  providers: RequestedSpeechProviders;
-  openaiConfig?: PaseoOpenAIConfig;
-  openAiCredentials: OpenAiCredentialState;
-  existing: {
-    sttService: SpeechToTextProvider | null;
-    ttsService: TextToSpeechProvider | null;
-    dictationSttService: SpeechToTextProvider | null;
-  };
-  logger: Logger;
-}): {
-  sttService: SpeechToTextProvider | null;
-  ttsService: TextToSpeechProvider | null;
-  dictationSttService: SpeechToTextProvider | null;
-} {
-  const { providers, openaiConfig, openAiCredentials, existing, logger } = params;
-
-  let sttService = existing.sttService;
-  let ttsService = existing.ttsService;
-  let dictationSttService = existing.dictationSttService;
-
-  const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai";
-  const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai";
-  const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai";
-
-  if (
-    (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
-    (openAiCredentials.openaiSttApiKey ||
-      openAiCredentials.openaiTtsApiKey ||
-      openAiCredentials.openaiDictationApiKey)
-  ) {
-    logger.info("OpenAI speech provider initialized");
-
-    if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) {
-      const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {};
-      sttService = new OpenAISTT(
-        {
-          apiKey: openAiCredentials.openaiSttApiKey,
-          ...sttConfig,
-        },
-        logger
-      );
-    }
-
-    if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) {
-      const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {};
-      ttsService = new OpenAITTS(
-        {
-          apiKey: openAiCredentials.openaiTtsApiKey,
-          voice: "alloy",
-          model: DEFAULT_OPENAI_TTS_MODEL,
-          responseFormat: "pcm",
-          ...ttsConfig,
-        },
-        logger
-      );
-    }
-
-    const dictationApiKey = openAiCredentials.openaiDictationApiKey;
-    if (needsOpenAiDictation && dictationApiKey) {
-      dictationSttService = {
-        id: "openai",
-        createSession: ({ logger: sessionLogger, language, prompt }) =>
-          new OpenAIRealtimeTranscriptionSession({
-            apiKey: dictationApiKey,
-            logger: sessionLogger,
-            transcriptionModel:
-              openaiConfig?.realtimeTranscriptionModel
-              ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
-            ...(language ? { language } : {}),
-            ...(prompt ? { prompt } : {}),
-            turnDetection: null,
-          }),
-      };
-    }
-  } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
-    logger.warn("OpenAI speech providers are configured but credentials are missing");
-  }
-
-  return {
-    sttService,
-    ttsService,
-    dictationSttService,
-  };
-}
-
 export type InitializedSpeechRuntime = {
   sttService: SpeechToTextProvider | null;
   ttsService: TextToSpeechProvider | null;
@@ -504,7 +31,7 @@ export type InitializedSpeechRuntime = {
   cleanup: () => void;
   localModelConfig: {
     modelsDir: string;
-    defaultModelIds: SherpaOnnxModelId[];
+    defaultModelIds: LocalSpeechModelId[];
   } | null;
 };
 
@@ -515,16 +42,12 @@ export async function initializeSpeechRuntime(params: {
 }): Promise<InitializedSpeechRuntime> {
   const logger = params.logger;
   const speechConfig = params.speechConfig ?? null;
-  const localConfig = speechConfig?.local ?? null;
   const openaiConfig = params.openaiConfig;
-
   const providers = resolveRequestedSpeechProviders(speechConfig);
-  const localModels = resolveConfiguredLocalModels(speechConfig);
-  const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
 
   validateOpenAiCredentialRequirements({
     providers,
-    openAiCredentials,
+    openaiConfig,
     logger,
   });
 
@@ -536,16 +59,8 @@ export async function initializeSpeechRuntime(params: {
         voiceTts: providers.voiceTtsProvider,
       },
       availability: {
-        openai: {
+        openai: getOpenAiSpeechAvailability(openaiConfig),
-          stt: Boolean(openAiCredentials.openaiSttApiKey),
+        local: getLocalSpeechAvailability(speechConfig),
-          tts: Boolean(openAiCredentials.openaiTtsApiKey),
-          dictationStt: Boolean(openAiCredentials.openaiDictationApiKey),
-        },
-        local: {
-          configured: Boolean(localConfig),
-          modelsDir: localConfig?.modelsDir ?? null,
-          autoDownload: localConfig?.autoDownload ?? null,
-        },
       },
     },
     "Speech provider reconciliation started"
@@ -553,8 +68,6 @@ export async function initializeSpeechRuntime(params: {
 
   const localSpeech = await initializeLocalSpeechServices({
     providers,
-    localConfig,
-    localModels,
     speechConfig,
     logger,
   });
@@ -562,7 +75,6 @@ export async function initializeSpeechRuntime(params: {
   const openAiSpeech = initializeOpenAiSpeechServices({
     providers,
     openaiConfig,
-    openAiCredentials,
     existing: {
       sttService: localSpeech.sttService,
       ttsService: localSpeech.ttsService,
@@ -610,27 +122,11 @@ export async function initializeSpeechRuntime(params: {
     "Speech provider reconciliation completed"
   );
 
-  const cleanup = () => {
-    const maybeFreeable = localSpeech.localVoiceTtsProvider as unknown as { free?: () => void } | null;
-    if (typeof maybeFreeable?.free === "function") {
-      maybeFreeable.free();
-    }
-    for (const engine of localSpeech.localSttEngines.values()) {
-      engine.engine.free();
-    }
-  };
-
   return {
     sttService: openAiSpeech.sttService,
     ttsService: openAiSpeech.ttsService,
     dictationSttService: openAiSpeech.dictationSttService,
-    cleanup,
+    cleanup: localSpeech.cleanup,
-    localModelConfig:
+    localModelConfig: localSpeech.localModelConfig,
-      localConfig
-        ? {
-            modelsDir: localConfig.modelsDir,
-            defaultModelIds: localSpeech.requiredLocalModelIds,
-          }
-        : null,
   };
 }

packages/server/src/server/speech/speech-types.ts

@@ -2,3 +2,9 @@ import { z } from "zod";
 
 export const SpeechProviderIdSchema = z.enum(["openai", "local"]);
 export type SpeechProviderId = z.infer<typeof SpeechProviderIdSchema>;
+
+export type RequestedSpeechProviders = {
+  dictationSttProvider: SpeechProviderId;
+  voiceSttProvider: SpeechProviderId;
+  voiceTtsProvider: SpeechProviderId;
+};