diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 4e62e14c3..04cfb27b6 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -41,20 +41,9 @@ function parseListenString(listen: string): ListenTarget { import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; -import { OpenAISTT, type STTConfig } from "./speech/providers/openai/stt.js"; -import { OpenAITTS, type TTSConfig } from "./speech/providers/openai/tts.js"; -import { OpenAIRealtimeTranscriptionSession } from "./speech/providers/openai/realtime-transcription-session.js"; -import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; -import { SherpaOnlineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-online-recognizer.js"; -import { SherpaOfflineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-offline-recognizer.js"; -import { SherpaOnnxSTT } from "./speech/providers/local/sherpa/sherpa-stt.js"; -import { SherpaOnnxParakeetSTT } from "./speech/providers/local/sherpa/sherpa-parakeet-stt.js"; -import { SherpaOnnxTTS } from "./speech/providers/local/sherpa/sherpa-tts.js"; -import { SherpaRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-realtime-session.js"; -import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-parakeet-realtime-session.js"; -import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/providers/local/sherpa/model-downloader.js"; -import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js"; -import { PocketTtsOnnxTTS } from "./speech/providers/local/pocket/pocket-tts-onnx.js"; +import type { STTConfig } from "./speech/providers/openai/stt.js"; +import type { TTSConfig } from "./speech/providers/openai/tts.js"; +import { initializeSpeechRuntime } from "./speech/speech-runtime.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; import { attachAgentStoragePersistence } from "./persistence-hooks.js"; @@ -108,26 +97,24 @@ export type PaseoOpenAIConfig = { apiKey?: string; stt?: Partial & { apiKey?: string }; tts?: Partial & { apiKey?: string }; + realtimeTranscriptionModel?: string; }; -export type PaseoSherpaOnnxConfig = { +export type PaseoLocalSpeechConfig = { modelsDir: string; autoDownload?: boolean; - stt?: { - preset?: string; - }; - tts?: { - preset?: string; - speakerId?: number; - speed?: number; - }; }; export type PaseoSpeechConfig = { dictationSttProvider?: "openai" | "local"; voiceSttProvider?: "openai" | "local"; voiceTtsProvider?: "openai" | "local"; - sherpaOnnx?: PaseoSherpaOnnxConfig; + local?: PaseoLocalSpeechConfig; + dictationLocalSttModel?: string; + voiceLocalSttModel?: string; + voiceLocalTtsModel?: string; + voiceLocalTtsSpeakerId?: number; + voiceLocalTtsSpeed?: number; }; export type PaseoDaemonConfig = { @@ -551,384 +538,17 @@ export async function createPaseoDaemon( }); }, }); - - - let sttService: SpeechToTextProvider | null = null; - let ttsService: TextToSpeechProvider | null = null; - let dictationSttService: SpeechToTextProvider | null = null; - - let sherpaOnline: SherpaOnlineRecognizerEngine | null = null; - let sherpaOffline: SherpaOfflineRecognizerEngine | null = null; - let sherpaTts: TextToSpeechProvider | null = null; - - const openaiApiKey = config.openai?.apiKey; - const speechConfig = config.speech ?? null; - const sherpaConfig = speechConfig?.sherpaOnnx ?? null; - - const voiceSttProvider = speechConfig?.voiceSttProvider ?? "local"; - const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "local"; - const dictationSttProvider = speechConfig?.dictationSttProvider ?? "local"; - - const wantsLocalDictation = dictationSttProvider === "local"; - const wantsLocalVoiceStt = voiceSttProvider === "local"; - const wantsLocalVoiceTts = voiceTtsProvider === "local"; - - const openaiSttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey; - const openaiTtsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey; - const openaiDictationApiKey = openaiApiKey; - - const missingOpenAiCredentialsFor: string[] = []; - if (voiceSttProvider === "openai" && !openaiSttApiKey) { - missingOpenAiCredentialsFor.push("voice.stt"); - } - if (voiceTtsProvider === "openai" && !openaiTtsApiKey) { - missingOpenAiCredentialsFor.push("voice.tts"); - } - if (dictationSttProvider === "openai" && !openaiDictationApiKey) { - missingOpenAiCredentialsFor.push("dictation.stt"); - } - - if (missingOpenAiCredentialsFor.length > 0) { - logger.error( - { - requestedProviders: { - dictationStt: dictationSttProvider, - voiceStt: voiceSttProvider, - voiceTts: voiceTtsProvider, - }, - missingOpenAiCredentialsFor, - }, - "Invalid speech configuration: OpenAI provider selected but credentials are missing" - ); - throw new Error( - `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` - ); - } - - logger.info( - { - requestedProviders: { - dictationStt: dictationSttProvider, - voiceStt: voiceSttProvider, - voiceTts: voiceTtsProvider, - }, - availability: { - openai: { - stt: Boolean(openaiSttApiKey), - tts: Boolean(openaiTtsApiKey), - dictationStt: Boolean(openaiDictationApiKey), - }, - local: { - configured: Boolean(sherpaConfig), - modelsDir: sherpaConfig?.modelsDir ?? null, - autoDownload: sherpaConfig?.autoDownload ?? null, - }, - }, - }, - "Speech provider reconciliation started" - ); - - if ((wantsLocalDictation || wantsLocalVoiceStt || wantsLocalVoiceTts) && sherpaConfig) { - const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true); - let sttPreset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim(); - if ( - sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" && - sttPreset !== "paraformer-bilingual-zh-en" && - sttPreset !== "parakeet-tdt-0.6b-v3-int8" - ) { - throw new Error(`Unknown local STT preset: ${sttPreset}`); - } - - let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); - if ( - ttsPreset !== "kitten-nano-en-v0_1-fp16" && - ttsPreset !== "kokoro-en-v0_19" && - ttsPreset !== "pocket-tts-onnx-int8" - ) { - throw new Error(`Unknown local TTS preset: ${ttsPreset}`); - } - - const modelIds: SherpaOnnxModelId[] = []; - if (wantsLocalDictation || wantsLocalVoiceStt) { - modelIds.push(sttPreset as SherpaOnnxModelId); - } - if (wantsLocalVoiceTts) { - modelIds.push(ttsPreset as SherpaOnnxModelId); - } - - try { - logger.info( - { - modelsDir: sherpaConfig.modelsDir, - modelIds, - autoDownload, - }, - "Ensuring local speech models" - ); - await ensureSherpaOnnxModels({ - modelsDir: sherpaConfig.modelsDir, - modelIds, - autoDownload, - logger, - }); - } catch (err) { - logger.error( - { - err, - modelsDir: sherpaConfig.modelsDir, - autoDownload, - hint: - "Run: npm run dev --workspace=@getpaseo/server, then run: " + - "`tsx packages/server/scripts/download-speech-models.ts --models-dir --model `", - }, - "Failed to ensure local speech models" - ); - } - } - - if ((wantsLocalDictation || wantsLocalVoiceStt) && sherpaConfig) { - let preset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim(); - if ( - preset !== "zipformer-bilingual-zh-en-2023-02-20" && - preset !== "paraformer-bilingual-zh-en" && - preset !== "parakeet-tdt-0.6b-v3-int8" - ) { - throw new Error(`Unknown local STT preset: ${preset}`); - } - const base = sherpaConfig.modelsDir; - - try { - if (preset === "parakeet-tdt-0.6b-v3-int8") { - const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8"); - sherpaOffline = new SherpaOfflineRecognizerEngine( - { - model: { - kind: "nemo_transducer", - encoder: `${modelDir}/encoder.int8.onnx`, - decoder: `${modelDir}/decoder.int8.onnx`, - joiner: `${modelDir}/joiner.int8.onnx`, - tokens: `${modelDir}/tokens.txt`, - }, - numThreads: 2, - debug: 0, - }, - logger - ); - } else { - const model = - preset === "paraformer-bilingual-zh-en" - ? { - kind: "paraformer" as const, - encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`, - decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`, - tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`, - } - : { - kind: "transducer" as const, - encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`, - decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`, - joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`, - tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`, - modelType: "zipformer", - }; - - sherpaOnline = new SherpaOnlineRecognizerEngine( - { - model, - numThreads: 1, - debug: 0, - }, - logger - ); - } - } catch (err) { - logger.error( - { - err, - modelsDir: sherpaConfig.modelsDir, - preset, - hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, - }, - "Failed to initialize Sherpa STT (models missing or invalid)" - ); - sherpaOnline = null; - sherpaOffline = null; - } - } else if (wantsLocalDictation || wantsLocalVoiceStt) { - logger.warn( - { configured: Boolean(sherpaConfig) }, - "Local STT selected but local provider config is missing; STT will be unavailable" - ); - } - - if (wantsLocalVoiceTts && sherpaConfig) { - let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); - if ( - preset !== "kitten-nano-en-v0_1-fp16" && - preset !== "kokoro-en-v0_19" && - preset !== "pocket-tts-onnx-int8" - ) { - throw new Error(`Unknown local TTS preset: ${preset}`); - } - try { - if (preset === "pocket-tts-onnx-int8") { - const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8"); - sherpaTts = await PocketTtsOnnxTTS.create( - { - modelDir, - precision: "int8", - targetChunkMs: 50, - }, - logger - ); - } else { - const modelDir = `${sherpaConfig.modelsDir}/${preset}`; - sherpaTts = new SherpaOnnxTTS( - { - preset: preset as any, - modelDir, - speakerId: sherpaConfig.tts?.speakerId, - speed: sherpaConfig.tts?.speed, - }, - logger - ); - } - } catch (err) { - logger.error( - { - err, - preset, - hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, - }, - "Failed to initialize Sherpa TTS (models missing or invalid)" - ); - sherpaTts = null; - } - } else if (wantsLocalVoiceTts) { - logger.warn( - { configured: Boolean(sherpaConfig) }, - "Local TTS selected but local provider config is missing; TTS will be unavailable" - ); - } - - if (wantsLocalVoiceStt && sherpaOffline) { - sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger); - } else if (wantsLocalVoiceStt && sherpaOnline) { - sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger); - } - - if (wantsLocalVoiceTts && sherpaTts) { - ttsService = sherpaTts; - } - - if (wantsLocalDictation && sherpaOnline) { - dictationSttService = { - id: "local", - createSession: () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }), - }; - } else if (wantsLocalDictation && sherpaOffline) { - dictationSttService = { - id: "local", - createSession: () => - new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }), - }; - } - - const needsOpenAiStt = !sttService && voiceSttProvider === "openai"; - const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai"; - const needsOpenAiDictation = dictationSttProvider === "openai"; - - if ( - (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && - (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey) - ) { - logger.info("OpenAI speech provider initialized"); - - if (needsOpenAiStt) { - if (openaiSttApiKey) { - const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {}; - sttService = new OpenAISTT( - { - apiKey: openaiSttApiKey, - ...sttConfig, - }, - logger - ); - } - } - - if (needsOpenAiTts) { - if (openaiTtsApiKey) { - const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {}; - ttsService = new OpenAITTS( - { - apiKey: openaiTtsApiKey, - voice: "alloy", - model: "tts-1", - responseFormat: "pcm", - ...ttsConfig, - }, - logger - ); - } - } - - if (needsOpenAiDictation) { - const transcriptionModel = - process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe"; - - dictationSttService = { - id: "openai", - createSession: ({ logger: sessionLogger, language, prompt }) => - new OpenAIRealtimeTranscriptionSession({ - apiKey: openaiDictationApiKey!, - logger: sessionLogger, - transcriptionModel, - ...(language ? { language } : {}), - ...(prompt ? { prompt } : {}), - turnDetection: null, - }), - }; - } - } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) { - logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation is unavailable"); - } - - const effectiveProviders = { - dictationStt: dictationSttService?.id ?? "unavailable", - voiceStt: sttService?.id ?? "unavailable", - voiceTts: !ttsService ? "unavailable" : ttsService === sherpaTts ? "local" : "openai", - }; - const unavailableFeatures = [ - !dictationSttService ? "dictation.stt" : null, - !sttService ? "voice.stt" : null, - !ttsService ? "voice.tts" : null, - ].filter((feature): feature is string => feature !== null); - - if (unavailableFeatures.length > 0) { - logger.error( - { - requestedProviders: { - dictationStt: dictationSttProvider, - voiceStt: voiceSttProvider, - voiceTts: voiceTtsProvider, - }, - effectiveProviders, - unavailableFeatures, - }, - "Speech provider reconciliation failed: configured features are unavailable" - ); - throw new Error( - `Configured speech features unavailable: ${unavailableFeatures.join(", ")}` - ); - } else { - logger.info( - { - effectiveProviders, - }, - "Speech provider reconciliation completed" - ); - } + const { + sttService, + ttsService, + dictationSttService, + cleanup: cleanupSpeechRuntime, + localModelConfig, + } = await initializeSpeechRuntime({ + logger, + openaiConfig: config.openai, + speechConfig: config.speech, + }); wsServer = new VoiceAssistantWebSocketServer( httpServer, @@ -962,6 +582,7 @@ export async function createPaseoDaemon( { finalTimeoutMs: config.dictationFinalTimeoutMs, stt: dictationSttService, + localModels: localModelConfig ?? undefined, } ); @@ -1064,11 +685,7 @@ export async function createPaseoDaemon( await agentStorage.flush().catch(() => undefined); await shutdownProviders(logger); terminalManager.killAll(); - if (sherpaTts && typeof (sherpaTts as any).free === "function") { - (sherpaTts as any).free(); - } - sherpaOnline?.free(); - sherpaOffline?.free(); + cleanupSpeechRuntime(); await relayTransport?.stop().catch(() => undefined); if (wsServer) { await wsServer.close(); diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index bbd455bdd..1b3283c2d 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -1,11 +1,10 @@ import path from "node:path"; import type { PaseoDaemonConfig } from "./bootstrap.js"; -import type { STTConfig } from "./speech/providers/openai/stt.js"; -import type { TTSConfig } from "./speech/providers/openai/tts.js"; import { loadPersistedConfig } from "./persisted-config.js"; import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; +import { resolveSpeechConfig } from "./speech/speech-config-resolver.js"; import { mergeAllowedHosts, parseAllowedHostsEnv, @@ -27,69 +26,6 @@ export type CliConfigOverrides = Partial<{ allowedHosts: AllowedHostsConfig; }>; -function parseOpenAIConfig( - env: NodeJS.ProcessEnv, - configApiKey: string | undefined, - config: { - dictationSttModel?: string; - dictationSttConfidenceThreshold?: number; - voiceSttModel?: string; - voiceTtsVoice?: TTSConfig["voice"]; - voiceTtsModel?: TTSConfig["model"]; - } -) { - const apiKey = env.OPENAI_API_KEY ?? configApiKey; - if (!apiKey) return undefined; - - const sttConfidenceThreshold = env.STT_CONFIDENCE_THRESHOLD - ? parseFloat(env.STT_CONFIDENCE_THRESHOLD) - : config.dictationSttConfidenceThreshold; - const sttModel = ( - env.STT_MODEL ?? - config.voiceSttModel ?? - config.dictationSttModel - ) as STTConfig["model"] | undefined; - const ttsVoice = (env.TTS_VOICE || "alloy") as - | "alloy" - | "echo" - | "fable" - | "onyx" - | "nova" - | "shimmer"; - const ttsModel = (env.TTS_MODEL || config.voiceTtsModel || "tts-1") as - | "tts-1" - | "tts-1-hd"; - const configuredVoice = config.voiceTtsVoice; - - return { - apiKey, - stt: { - apiKey, - confidenceThreshold: sttConfidenceThreshold, - ...(sttModel ? { model: sttModel } : {}), - }, - tts: { - apiKey, - voice: configuredVoice ?? ttsVoice, - model: ttsModel, - responseFormat: "pcm" as TTSConfig["responseFormat"], - }, - }; -} - -function parseSpeechProviderId(value: unknown): "openai" | "local" | null { - if (typeof value !== "string") { - return null; - } - const normalized = value.trim().toLowerCase(); - if (!normalized) { - return null; - } - if (normalized === "openai") return "openai"; - if (normalized === "local") return "local"; - return null; -} - function parseVoiceLlmProviderId(value: unknown): AgentProvider | null { if (typeof value !== "string") { return null; @@ -103,36 +39,6 @@ function parseVoiceLlmProviderId(value: unknown): AgentProvider | null { : null; } -function normalizeSherpaSttPreset(value: string): string { - const raw = value.trim(); - const normalized = raw.toLowerCase(); - if (normalized === "zipformer" || normalized === "zipformer-bilingual") { - return "zipformer-bilingual-zh-en-2023-02-20"; - } - if (normalized === "paraformer") { - return "paraformer-bilingual-zh-en"; - } - if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") { - return "parakeet-tdt-0.6b-v3-int8"; - } - return raw; -} - -function normalizeSherpaTtsPreset(value: string): string { - const raw = value.trim(); - const normalized = raw.toLowerCase(); - if (normalized === "pocket" || normalized === "pocket-tts") { - return "pocket-tts-onnx-int8"; - } - if (normalized === "kitten") { - return "kitten-nano-en-v0_1-fp16"; - } - if (normalized === "kokoro") { - return "kokoro-en-v0_19"; - } - return raw; -} - export function loadConfig( paseoHome: string, options?: { @@ -185,79 +91,12 @@ export function loadConfig( const appBaseUrl = env.PASEO_APP_BASE_URL ?? persisted.app?.baseUrl ?? DEFAULT_APP_BASE_URL; - const openai = parseOpenAIConfig(env, persisted.providers?.openai?.apiKey, { - dictationSttModel: persisted.features?.dictation?.stt?.model, - dictationSttConfidenceThreshold: - persisted.features?.dictation?.stt?.confidenceThreshold, - voiceSttModel: persisted.features?.voiceMode?.stt?.model, - voiceTtsModel: persisted.features?.voiceMode?.tts?.model, - voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice, + const { openai, speech } = resolveSpeechConfig({ + paseoHome, + env, + persisted, }); - const dictationSttProvider = - parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ?? - parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ?? - "local"; - - const voiceSttProvider = - parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ?? - parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ?? - "local"; - - const voiceTtsProvider = - parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ?? - parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ?? - "local"; - - const shouldConfigureSherpa = - dictationSttProvider === "local" || - voiceSttProvider === "local" || - voiceTtsProvider === "local" || - typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" || - Boolean(persisted.providers?.sherpaOnnx); - - const sherpaModelsDir = - (env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() || - path.join(paseoHome, "models", "sherpa-onnx"); - - const sherpaOnnx = shouldConfigureSherpa - ? { - modelsDir: sherpaModelsDir, - autoDownload: - env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined - ? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1" - : persisted.providers?.sherpaOnnx?.autoDownload ?? - // In tests we should never hit the network unexpectedly. - Boolean(env.VITEST) === false, - stt: { - preset: normalizeSherpaSttPreset( - (env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() || - (persisted.features?.voiceMode?.stt?.preset ?? - persisted.features?.dictation?.stt?.preset)?.trim() || - "parakeet-tdt-0.6b-v3-int8" - ), - }, - tts: { - preset: normalizeSherpaTtsPreset( - (env.PASEO_SHERPA_TTS_PRESET ?? - persisted.providers?.sherpaOnnx?.tts?.preset ?? - persisted.features?.voiceMode?.tts?.preset)?.trim() || - (env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8") - ), - speakerId: - env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined - ? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10) - : persisted.providers?.sherpaOnnx?.tts?.speakerId ?? - persisted.features?.voiceMode?.tts?.speakerId, - speed: - env.PASEO_SHERPA_TTS_SPEED !== undefined - ? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED) - : persisted.providers?.sherpaOnnx?.tts?.speed ?? - persisted.features?.voiceMode?.tts?.speed, - }, - } - : undefined; - const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER); const persistedVoiceLlmProvider = parseVoiceLlmProviderId( persisted.features?.voiceMode?.llm?.provider @@ -284,12 +123,7 @@ export function loadConfig( relayPublicEndpoint, appBaseUrl, openai, - speech: { - dictationSttProvider, - voiceSttProvider, - voiceTtsProvider, - ...(sherpaOnnx ? { sherpaOnnx } : {}), - }, + speech, voiceLlmProvider, voiceLlmProviderExplicit, voiceLlmModel, diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index b395c1191..3941ecb0f 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -18,9 +18,9 @@ import { const openaiApiKey = process.env.OPENAI_API_KEY ?? null; -const sherpaModelsDir = - process.env.PASEO_SHERPA_ONNX_MODELS_DIR ?? - path.join(homedir(), ".paseo", "models", "sherpa-onnx"); +const localModelsDir = + process.env.PASEO_LOCAL_MODELS_DIR ?? + path.join(homedir(), ".paseo", "models", "local-speech"); function hasSherpaZipformerModels(modelsDir: string): boolean { return ( @@ -49,7 +49,7 @@ function hasSherpaKittenModels(modelsDir: string): boolean { ); } -const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir); +const hasLocalSpeech = hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir); const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey); const speechTest = hasAnySpeech ? test : test.skip; @@ -102,15 +102,17 @@ describe("daemon client E2E", () => { dictationSttProvider: "local" as const, voiceSttProvider: "local" as const, voiceTtsProvider: "local" as const, - sherpaOnnx: { - modelsDir: sherpaModelsDir, - stt: { - preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", - }, - tts: { - preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16", - }, + local: { + modelsDir: localModelsDir, }, + dictationLocalSttModel: + process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceLocalSttModel: + process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceLocalTtsModel: + process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16", }; ctx = await createDaemonTestContext({ diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 2ee622536..690e8bef3 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -18,31 +18,17 @@ const ProviderCredentialsSchema = z }) .strict(); -const SherpaOnnxProviderSchema = z +const LocalSpeechProviderSchema = z .object({ modelsDir: z.string().min(1).optional(), autoDownload: z.boolean().optional(), - stt: z - .object({ - preset: z.string().min(1).optional(), - }) - .strict() - .optional(), - tts: z - .object({ - preset: z.string().min(1).optional(), - speakerId: z.number().int().optional(), - speed: z.number().optional(), - }) - .strict() - .optional(), }) .strict(); const ProvidersSchema = z .object({ openai: ProviderCredentialsSchema.optional(), - sherpaOnnx: SherpaOnnxProviderSchema.optional(), + local: LocalSpeechProviderSchema.optional(), }) .strict(); @@ -62,7 +48,6 @@ const FeatureDictationSchema = z .object({ provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), - preset: z.string().min(1).optional(), confidenceThreshold: z.number().optional(), }) .strict() @@ -83,16 +68,14 @@ const FeatureVoiceModeSchema = z .object({ provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), - preset: z.string().min(1).optional(), }) .strict() .optional(), tts: z .object({ provider: SpeechProviderIdSchema.optional(), - model: z.enum(["tts-1", "tts-1-hd"]).optional(), + model: z.string().min(1).optional(), voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(), - preset: z.string().min(1).optional(), speakerId: z.number().int().optional(), speed: z.number().optional(), }) diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 7249e4d9e..ff4c87e1d 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -339,6 +339,8 @@ export class Session { private readonly voiceLlmProviderExplicit: boolean; private readonly voiceLlmModel: string | null; private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null; + private readonly localSpeechModelsDir: string; + private readonly defaultLocalSpeechModelIds: SherpaOnnxModelId[]; private readonly registerVoiceSpeakHandler?: ( agentId: string, handler: VoiceSpeakHandler @@ -380,6 +382,10 @@ export class Session { dictation?: { finalTimeoutMs?: number; stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + }; } ) { this.clientId = clientId; @@ -397,6 +403,15 @@ export class Session { this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false; this.voiceLlmModel = voice?.voiceLlmModel ?? null; this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null; + const configuredModelsDir = dictation?.localModels?.modelsDir?.trim(); + this.localSpeechModelsDir = + configuredModelsDir && configuredModelsDir.length > 0 + ? configuredModelsDir + : join(this.paseoHome, "models", "local-speech"); + this.defaultLocalSpeechModelIds = + dictation?.localModels?.defaultModelIds && dictation.localModels.defaultModelIds.length > 0 + ? [...new Set(dictation.localModels.defaultModelIds)] + : ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"]; this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler; this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler; this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext; @@ -1957,9 +1972,7 @@ export class Session { private async handleSpeechModelsListRequest( msg: Extract ): Promise { - const modelsDir = - process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() || - join(this.paseoHome, "models", "sherpa-onnx"); + const modelsDir = this.localSpeechModelsDir; const models = await Promise.all( listSherpaOnnxModels().map(async (model) => { @@ -2004,17 +2017,12 @@ export class Session { private async handleSpeechModelsDownloadRequest( msg: Extract ): Promise { - const modelsDir = - process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() || - join(this.paseoHome, "models", "sherpa-onnx"); + const modelsDir = this.localSpeechModelsDir; const modelIdsRaw = msg.modelIds && msg.modelIds.length > 0 ? msg.modelIds - : [ - process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", - process.env.PASEO_SHERPA_TTS_PRESET ?? "pocket-tts-onnx-int8", - ]; + : this.defaultLocalSpeechModelIds; const allModelIds = new Set(listSherpaOnnxModels().map((m) => m.id)); const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as SherpaOnnxModelId)); diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 94c6d62bd..6e679d6db 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts"; export type SherpaOnnxModelId = @@ -8,6 +10,50 @@ export type SherpaOnnxModelId = | "kokoro-en-v0_19" | "pocket-tts-onnx-int8"; +export const LOCAL_STT_MODEL_IDS = [ + "zipformer-bilingual-zh-en-2023-02-20", + "paraformer-bilingual-zh-en", + "parakeet-tdt-0.6b-v3-int8", +] as const; +export type LocalSttModelId = (typeof LOCAL_STT_MODEL_IDS)[number]; + +export const LOCAL_TTS_MODEL_IDS = [ + "kitten-nano-en-v0_1-fp16", + "kokoro-en-v0_19", + "pocket-tts-onnx-int8", +] as const; +export type LocalTtsModelId = (typeof LOCAL_TTS_MODEL_IDS)[number]; + +const STT_MODEL_ALIASES: Record = { + zipformer: "zipformer-bilingual-zh-en-2023-02-20", + "zipformer-bilingual": "zipformer-bilingual-zh-en-2023-02-20", + paraformer: "paraformer-bilingual-zh-en", + parakeet: "parakeet-tdt-0.6b-v3-int8", + "parakeet-v3": "parakeet-tdt-0.6b-v3-int8", + "parakeet-tdt": "parakeet-tdt-0.6b-v3-int8", +}; + +const TTS_MODEL_ALIASES: Record = { + pocket: "pocket-tts-onnx-int8", + "pocket-tts": "pocket-tts-onnx-int8", + kitten: "kitten-nano-en-v0_1-fp16", + kokoro: "kokoro-en-v0_19", +}; + +export const LocalSttModelIdSchema = z.preprocess((value) => { + if (typeof value !== "string") return value; + const normalized = value.trim().toLowerCase(); + if (!normalized) return value; + return STT_MODEL_ALIASES[normalized] ?? normalized; +}, z.enum(LOCAL_STT_MODEL_IDS)); + +export const LocalTtsModelIdSchema = z.preprocess((value) => { + if (typeof value !== "string") return value; + const normalized = value.trim().toLowerCase(); + if (!normalized) return value; + return TTS_MODEL_ALIASES[normalized] ?? normalized; +}, z.enum(LOCAL_TTS_MODEL_IDS)); + export type SherpaOnnxModelSpec = { id: SherpaOnnxModelId; kind: SherpaOnnxModelKind; diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts index 9c434c4be..68bbe83a6 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts @@ -110,7 +110,7 @@ export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOption if (!options.autoDownload) { throw new Error( `Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` + - `Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.` + `Set PASEO_LOCAL_AUTO_DOWNLOAD=1 to auto-download.` ); } diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index 8d66efdc5..01bf369ea 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -96,7 +96,7 @@ describe("speech models (download E2E)", () => { const set = getModelSet(); const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-")); - const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx"); + const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech"); const modelIds: SherpaOnnxModelId[] = set === "parakeet-pocket" @@ -117,12 +117,20 @@ describe("speech models (download E2E)", () => { dictationSttProvider: "local", voiceSttProvider: "local", voiceTtsProvider: "local", - sherpaOnnx: { + local: { modelsDir, autoDownload: false, - stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" }, - tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" }, }, + dictationLocalSttModel: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceLocalSttModel: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceLocalTtsModel: + set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16", }, }); diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts new file mode 100644 index 000000000..bb34fdb01 --- /dev/null +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -0,0 +1,94 @@ +import path from "node:path"; + +import { describe, expect, test } from "vitest"; + +import { PersistedConfigSchema } from "../persisted-config.js"; +import { resolveSpeechConfig } from "./speech-config-resolver.js"; + +describe("resolveSpeechConfig", () => { + test("resolves local-first defaults without env overrides", () => { + const paseoHome = "/tmp/paseo-home"; + const persisted = PersistedConfigSchema.parse({}); + const env = {} as NodeJS.ProcessEnv; + + const result = resolveSpeechConfig({ + paseoHome, + env, + persisted, + }); + + expect(result.openai).toBeUndefined(); + expect(result.speech.dictationSttProvider).toBe("local"); + expect(result.speech.voiceSttProvider).toBe("local"); + expect(result.speech.voiceTtsProvider).toBe("local"); + expect(result.speech.local).toEqual({ + modelsDir: path.join(paseoHome, "models", "local-speech"), + autoDownload: true, + }); + expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8"); + }); + + test("resolves feature-scoped local model env vars", () => { + const persisted = PersistedConfigSchema.parse({ + features: { + voiceMode: { + stt: { provider: "openai", model: "gpt-4o-transcribe" }, + }, + }, + providers: { + openai: { apiKey: "persisted-key" }, + }, + }); + const env = { + PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer", + PASEO_VOICE_LOCAL_STT_MODEL: "parakeet", + PASEO_VOICE_LOCAL_TTS_MODEL: "kitten", + PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5", + PASEO_VOICE_LOCAL_TTS_SPEED: "1.35", + PASEO_LOCAL_MODELS_DIR: "/tmp/models", + PASEO_LOCAL_AUTO_DOWNLOAD: "0", + OPENAI_API_KEY: "env-key", + PASEO_VOICE_STT_PROVIDER: "openai", + PASEO_DICTATION_STT_PROVIDER: "local", + PASEO_VOICE_TTS_PROVIDER: "local", + } as NodeJS.ProcessEnv; + + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env, + persisted, + }); + + expect(result.speech.local).toEqual({ + modelsDir: "/tmp/models", + autoDownload: false, + }); + expect(result.speech.dictationLocalSttModel).toBe("zipformer-bilingual-zh-en-2023-02-20"); + expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalTtsModel).toBe("kitten-nano-en-v0_1-fp16"); + expect(result.speech.voiceLocalTtsSpeakerId).toBe(5); + expect(result.speech.voiceLocalTtsSpeed).toBe(1.35); + expect(result.openai?.apiKey).toBe("env-key"); + expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe"); + }); + + test("ignores deprecated shared local model env vars", () => { + const persisted = PersistedConfigSchema.parse({}); + const env = { + PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20", + PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16", + } as NodeJS.ProcessEnv; + + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env, + persisted, + }); + + expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8"); + }); +}); diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts new file mode 100644 index 000000000..964f5b8df --- /dev/null +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -0,0 +1,196 @@ +import path from "node:path"; + +import type { STTConfig } from "./providers/openai/stt.js"; +import type { TTSConfig } from "./providers/openai/tts.js"; +import type { PersistedConfig } from "../persisted-config.js"; +import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; +import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/sherpa/model-catalog.js"; + +const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8"; +const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8"; +const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech"); +const DEFAULT_OPENAI_TTS_MODEL: TTSConfig["model"] = "tts-1"; +const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; + +type SpeechProviderId = "openai" | "local"; + +function parseSpeechProviderId(value: unknown): SpeechProviderId | null { + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + if (!normalized) return null; + if (normalized === "openai") return "openai"; + if (normalized === "local") return "local"; + return null; +} + +function parseBooleanFlag(value: string | undefined): boolean | null { + if (value === undefined) return null; + const normalized = value.trim().toLowerCase(); + if (normalized === "1" || normalized === "true" || normalized === "yes") return true; + if (normalized === "0" || normalized === "false" || normalized === "no") return false; + return null; +} + +function parseNumberOrUndefined(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const parsed = Number.parseFloat(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function parseIntOrUndefined(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function parseOpenAiConfig( + env: NodeJS.ProcessEnv, + persisted: PersistedConfig, + providers: { + dictationSttProvider: SpeechProviderId; + voiceSttProvider: SpeechProviderId; + voiceTtsProvider: SpeechProviderId; + } +): PaseoOpenAIConfig | undefined { + const apiKey = env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey; + if (!apiKey) return undefined; + + const sttConfidenceThreshold = parseNumberOrUndefined(env.STT_CONFIDENCE_THRESHOLD) + ?? persisted.features?.dictation?.stt?.confidenceThreshold; + + const sttModel = ( + env.STT_MODEL + ?? (providers.voiceSttProvider === "openai" ? persisted.features?.voiceMode?.stt?.model : undefined) + ?? (providers.dictationSttProvider === "openai" ? persisted.features?.dictation?.stt?.model : undefined) + ) as STTConfig["model"] | undefined; + + const ttsVoice = ( + env.TTS_VOICE + || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.voice : undefined) + || "alloy" + ) as TTSConfig["voice"]; + const ttsModelRaw = + env.TTS_MODEL + || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.model : undefined) + || DEFAULT_OPENAI_TTS_MODEL; + const ttsModel: TTSConfig["model"] = + ttsModelRaw === "tts-1" || ttsModelRaw === "tts-1-hd" ? ttsModelRaw : DEFAULT_OPENAI_TTS_MODEL; + + const realtimeTranscriptionModel = + env.OPENAI_REALTIME_TRANSCRIPTION_MODEL + || (providers.dictationSttProvider === "openai" + ? persisted.features?.dictation?.stt?.model + : undefined) + || DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL; + + return { + apiKey, + stt: { + apiKey, + ...(sttConfidenceThreshold !== undefined ? { confidenceThreshold: sttConfidenceThreshold } : {}), + ...(sttModel ? { model: sttModel } : {}), + }, + tts: { + apiKey, + voice: ttsVoice, + model: ttsModel, + responseFormat: "pcm", + }, + realtimeTranscriptionModel, + }; +} + +export function resolveSpeechConfig(params: { + paseoHome: string; + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; +}): { + openai: PaseoOpenAIConfig | undefined; + speech: PaseoSpeechConfig; +} { + const { paseoHome, env, persisted } = params; + + const dictationSttProvider = + parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) + ?? parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) + ?? "local"; + + const voiceSttProvider = + parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) + ?? parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) + ?? "local"; + + const voiceTtsProvider = + parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) + ?? parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) + ?? "local"; + + const anyLocalRequested = + dictationSttProvider === "local" || + voiceSttProvider === "local" || + voiceTtsProvider === "local" || + env.PASEO_LOCAL_MODELS_DIR !== undefined || + persisted.providers?.local !== undefined; + + const localModelsDir = + env.PASEO_LOCAL_MODELS_DIR + ?? persisted.providers?.local?.modelsDir + ?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR); + + const localAutoDownload = + parseBooleanFlag(env.PASEO_LOCAL_AUTO_DOWNLOAD) + ?? persisted.providers?.local?.autoDownload + ?? true; + + const dictationLocalSttModel = LocalSttModelIdSchema.parse( + env.PASEO_DICTATION_LOCAL_STT_MODEL + ?? persisted.features?.dictation?.stt?.model + ?? DEFAULT_LOCAL_STT_MODEL + ); + + const voiceLocalSttModel = LocalSttModelIdSchema.parse( + env.PASEO_VOICE_LOCAL_STT_MODEL + ?? persisted.features?.voiceMode?.stt?.model + ?? DEFAULT_LOCAL_STT_MODEL + ); + + const voiceLocalTtsModel = LocalTtsModelIdSchema.parse( + env.PASEO_VOICE_LOCAL_TTS_MODEL + ?? persisted.features?.voiceMode?.tts?.model + ?? DEFAULT_LOCAL_TTS_MODEL + ); + + const voiceLocalTtsSpeakerId = + parseIntOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID) + ?? persisted.features?.voiceMode?.tts?.speakerId; + + const voiceLocalTtsSpeed = + parseNumberOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEED) + ?? persisted.features?.voiceMode?.tts?.speed; + + return { + openai: parseOpenAiConfig(env, persisted, { + dictationSttProvider, + voiceSttProvider, + voiceTtsProvider, + }), + speech: { + dictationSttProvider, + voiceSttProvider, + voiceTtsProvider, + ...(anyLocalRequested + ? { + local: { + modelsDir: localModelsDir.trim(), + autoDownload: localAutoDownload, + }, + } + : {}), + dictationLocalSttModel, + voiceLocalSttModel, + voiceLocalTtsModel, + ...(voiceLocalTtsSpeakerId !== undefined ? { voiceLocalTtsSpeakerId } : {}), + ...(voiceLocalTtsSpeed !== undefined ? { voiceLocalTtsSpeed } : {}), + }, + }; +} diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts new file mode 100644 index 000000000..0349b8291 --- /dev/null +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -0,0 +1,529 @@ +import type { Logger } from "pino"; + +import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; +import { + OpenAISTT, +} from "./providers/openai/stt.js"; +import { + OpenAITTS, +} from "./providers/openai/tts.js"; +import { OpenAIRealtimeTranscriptionSession } from "./providers/openai/realtime-transcription-session.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js"; +import { SherpaOnlineRecognizerEngine } from "./providers/local/sherpa/sherpa-online-recognizer.js"; +import { SherpaOfflineRecognizerEngine } from "./providers/local/sherpa/sherpa-offline-recognizer.js"; +import { SherpaOnnxSTT } from "./providers/local/sherpa/sherpa-stt.js"; +import { SherpaOnnxParakeetSTT } from "./providers/local/sherpa/sherpa-parakeet-stt.js"; +import { SherpaOnnxTTS } from "./providers/local/sherpa/sherpa-tts.js"; +import { SherpaRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-realtime-session.js"; +import { SherpaParakeetRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-parakeet-realtime-session.js"; +import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./providers/local/sherpa/model-downloader.js"; +import { + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + type LocalSttModelId, + type LocalTtsModelId, + type SherpaOnnxModelId, +} from "./providers/local/sherpa/model-catalog.js"; +import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js"; + +type SpeechProviderId = "openai" | "local"; + +const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8"; +const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8"; +const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; + +type LocalSttEngine = + | { kind: "offline"; engine: SherpaOfflineRecognizerEngine } + | { kind: "online"; engine: SherpaOnlineRecognizerEngine }; + +function buildModelDownloadHint(modelsDir: string, modelId: SherpaOnnxModelId): string { + return `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${modelsDir}' --model '${modelId}'`; +} + +function resolveSpeechProviders( + speechConfig: PaseoSpeechConfig | null +): { + dictationSttProvider: SpeechProviderId; + voiceSttProvider: SpeechProviderId; + voiceTtsProvider: SpeechProviderId; +} { + return { + dictationSttProvider: speechConfig?.dictationSttProvider ?? "local", + voiceSttProvider: speechConfig?.voiceSttProvider ?? "local", + voiceTtsProvider: speechConfig?.voiceTtsProvider ?? "local", + }; +} + +function resolveLocalModels( + speechConfig: PaseoSpeechConfig | null +): { + dictationLocalSttModel: LocalSttModelId; + voiceLocalSttModel: LocalSttModelId; + voiceLocalTtsModel: LocalTtsModelId; +} { + return { + dictationLocalSttModel: LocalSttModelIdSchema.parse( + speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL + ), + voiceLocalSttModel: LocalSttModelIdSchema.parse( + speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL + ), + voiceLocalTtsModel: LocalTtsModelIdSchema.parse( + speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL + ), + }; +} + +function computeDefaultLocalModelIds(params: { + providers: { + dictationSttProvider: SpeechProviderId; + voiceSttProvider: SpeechProviderId; + voiceTtsProvider: SpeechProviderId; + }; + models: { + dictationLocalSttModel: SherpaOnnxModelId; + voiceLocalSttModel: SherpaOnnxModelId; + voiceLocalTtsModel: SherpaOnnxModelId; + }; +}): SherpaOnnxModelId[] { + const ids = new Set(); + if (params.providers.dictationSttProvider === "local") { + ids.add(params.models.dictationLocalSttModel); + } + if (params.providers.voiceSttProvider === "local") { + ids.add(params.models.voiceLocalSttModel); + } + if (params.providers.voiceTtsProvider === "local") { + ids.add(params.models.voiceLocalTtsModel); + } + return Array.from(ids); +} + +async function createLocalSttEngine(params: { + modelId: LocalSttModelId; + modelsDir: string; + logger: Logger; +}): Promise { + const { modelId, modelsDir, logger } = params; + + if (modelId === "parakeet-tdt-0.6b-v3-int8") { + const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + return { + kind: "offline", + engine: new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + joiner: `${modelDir}/joiner.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 2, + debug: 0, + }, + logger + ), + }; + } + + if (modelId === "paraformer-bilingual-zh-en") { + const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + return { + kind: "online", + engine: new SherpaOnlineRecognizerEngine( + { + model: { + kind: "paraformer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 1, + debug: 0, + }, + logger + ), + }; + } + + if (modelId === "zipformer-bilingual-zh-en-2023-02-20") { + const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + return { + kind: "online", + engine: new SherpaOnlineRecognizerEngine( + { + model: { + kind: "transducer", + encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`, + decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`, + joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`, + tokens: `${modelDir}/tokens.txt`, + modelType: "zipformer", + }, + numThreads: 1, + debug: 0, + }, + logger + ), + }; + } + + throw new Error(`Unsupported local STT model '${modelId}'`); +} + +export type InitializedSpeechRuntime = { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; + cleanup: () => void; + localModelConfig: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + } | null; +}; + +export async function initializeSpeechRuntime(params: { + logger: Logger; + openaiConfig?: PaseoOpenAIConfig; + speechConfig?: PaseoSpeechConfig; +}): Promise { + const logger = params.logger; + const speechConfig = params.speechConfig ?? null; + const localConfig = speechConfig?.local ?? null; + const openaiConfig = params.openaiConfig; + + const providers = resolveSpeechProviders(speechConfig); + const localModels = resolveLocalModels(speechConfig); + + const wantsLocalDictation = providers.dictationSttProvider === "local"; + const wantsLocalVoiceStt = providers.voiceSttProvider === "local"; + const wantsLocalVoiceTts = providers.voiceTtsProvider === "local"; + + const openaiApiKey = openaiConfig?.apiKey; + const openaiSttApiKey = openaiConfig?.stt?.apiKey ?? openaiApiKey; + const openaiTtsApiKey = openaiConfig?.tts?.apiKey ?? openaiApiKey; + const openaiDictationApiKey = openaiApiKey; + + const missingOpenAiCredentialsFor: string[] = []; + if (providers.voiceSttProvider === "openai" && !openaiSttApiKey) { + missingOpenAiCredentialsFor.push("voice.stt"); + } + if (providers.voiceTtsProvider === "openai" && !openaiTtsApiKey) { + missingOpenAiCredentialsFor.push("voice.tts"); + } + if (providers.dictationSttProvider === "openai" && !openaiDictationApiKey) { + missingOpenAiCredentialsFor.push("dictation.stt"); + } + if (missingOpenAiCredentialsFor.length > 0) { + logger.error( + { + requestedProviders: { + dictationStt: providers.dictationSttProvider, + voiceStt: providers.voiceSttProvider, + voiceTts: providers.voiceTtsProvider, + }, + missingOpenAiCredentialsFor, + }, + "Invalid speech configuration: OpenAI provider selected but credentials are missing" + ); + throw new Error( + `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` + ); + } + + logger.info( + { + requestedProviders: { + dictationStt: providers.dictationSttProvider, + voiceStt: providers.voiceSttProvider, + voiceTts: providers.voiceTtsProvider, + }, + availability: { + openai: { + stt: Boolean(openaiSttApiKey), + tts: Boolean(openaiTtsApiKey), + dictationStt: Boolean(openaiDictationApiKey), + }, + local: { + configured: Boolean(localConfig), + modelsDir: localConfig?.modelsDir ?? null, + autoDownload: localConfig?.autoDownload ?? null, + }, + }, + }, + "Speech provider reconciliation started" + ); + + let sttService: SpeechToTextProvider | null = null; + let ttsService: TextToSpeechProvider | null = null; + let dictationSttService: SpeechToTextProvider | null = null; + let localVoiceTtsProvider: TextToSpeechProvider | null = null; + + const requiredLocalModelIds = computeDefaultLocalModelIds({ + providers, + models: localModels, + }); + + if (requiredLocalModelIds.length > 0 && localConfig) { + try { + logger.info( + { + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + }, + "Ensuring local speech models" + ); + await ensureSherpaOnnxModels({ + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + logger, + }); + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + hint: + "Run: npm run dev --workspace=@getpaseo/server, then run: " + + "`tsx packages/server/scripts/download-speech-models.ts --models-dir --model `", + }, + "Failed to ensure local speech models" + ); + } + } + + const localSttEngines = new Map(); + const getLocalSttEngine = async ( + modelId: LocalSttModelId + ): Promise => { + const existing = localSttEngines.get(modelId); + if (existing) { + return existing; + } + if (!localConfig) { + return null; + } + try { + const created = await createLocalSttEngine({ + modelId, + modelsDir: localConfig.modelsDir, + logger, + }); + localSttEngines.set(modelId, created); + return created; + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelId, + hint: buildModelDownloadHint(localConfig.modelsDir, modelId), + }, + "Failed to initialize local STT engine (models missing or invalid)" + ); + return null; + } + }; + + if (wantsLocalVoiceStt) { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local STT selected for voice but local provider config is missing; STT will be unavailable" + ); + } else { + const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel); + if (voiceEngine?.kind === "offline") { + sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger); + } else if (voiceEngine?.kind === "online") { + sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger); + } + } + } + + if (wantsLocalDictation) { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable" + ); + } else { + const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel); + if (dictationEngine?.kind === "offline") { + dictationSttService = { + id: "local", + createSession: () => + new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }), + }; + } else if (dictationEngine?.kind === "online") { + dictationSttService = { + id: "local", + createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }), + }; + } + } + } + + if (wantsLocalVoiceTts) { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local TTS selected for voice but local provider config is missing; TTS will be unavailable" + ); + } else { + try { + if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") { + const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + localVoiceTtsProvider = await PocketTtsOnnxTTS.create( + { + modelDir, + precision: "int8", + targetChunkMs: 50, + }, + logger + ); + } else { + const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + localVoiceTtsProvider = new SherpaOnnxTTS( + { + preset: localModels.voiceLocalTtsModel, + modelDir, + speakerId: speechConfig?.voiceLocalTtsSpeakerId, + speed: speechConfig?.voiceLocalTtsSpeed, + }, + logger + ); + } + ttsService = localVoiceTtsProvider; + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelId: localModels.voiceLocalTtsModel, + hint: buildModelDownloadHint(localConfig.modelsDir, localModels.voiceLocalTtsModel), + }, + "Failed to initialize local TTS engine (models missing or invalid)" + ); + } + } + } + + const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai"; + const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai"; + const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai"; + + if ( + (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && + (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey) + ) { + logger.info("OpenAI speech provider initialized"); + + if (needsOpenAiStt && openaiSttApiKey) { + const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {}; + sttService = new OpenAISTT( + { + apiKey: openaiSttApiKey, + ...sttConfig, + }, + logger + ); + } + + if (needsOpenAiTts && openaiTtsApiKey) { + const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {}; + ttsService = new OpenAITTS( + { + apiKey: openaiTtsApiKey, + voice: "alloy", + model: "tts-1", + responseFormat: "pcm", + ...ttsConfig, + }, + logger + ); + } + + if (needsOpenAiDictation && openaiDictationApiKey) { + dictationSttService = { + id: "openai", + createSession: ({ logger: sessionLogger, language, prompt }) => + new OpenAIRealtimeTranscriptionSession({ + apiKey: openaiDictationApiKey, + logger: sessionLogger, + transcriptionModel: + openaiConfig?.realtimeTranscriptionModel + ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + ...(language ? { language } : {}), + ...(prompt ? { prompt } : {}), + turnDetection: null, + }), + }; + } + } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) { + logger.warn("OpenAI speech providers are configured but credentials are missing"); + } + + const effectiveProviders = { + dictationStt: dictationSttService?.id ?? "unavailable", + voiceStt: sttService?.id ?? "unavailable", + voiceTts: !ttsService ? "unavailable" : ttsService === localVoiceTtsProvider ? "local" : "openai", + }; + const unavailableFeatures = [ + !dictationSttService ? "dictation.stt" : null, + !sttService ? "voice.stt" : null, + !ttsService ? "voice.tts" : null, + ].filter((feature): feature is string => feature !== null); + + if (unavailableFeatures.length > 0) { + logger.error( + { + requestedProviders: { + dictationStt: providers.dictationSttProvider, + voiceStt: providers.voiceSttProvider, + voiceTts: providers.voiceTtsProvider, + }, + effectiveProviders, + unavailableFeatures, + }, + "Speech provider reconciliation failed: configured features are unavailable" + ); + throw new Error(`Configured speech features unavailable: ${unavailableFeatures.join(", ")}`); + } + + logger.info( + { + effectiveProviders, + }, + "Speech provider reconciliation completed" + ); + + const cleanup = () => { + const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null; + if (typeof maybeFreeable?.free === "function") { + maybeFreeable.free(); + } + for (const engine of localSttEngines.values()) { + engine.engine.free(); + } + }; + + return { + sttService, + ttsService, + dictationSttService, + cleanup, + localModelConfig: + localConfig + ? { + modelsDir: localConfig.modelsDir, + defaultModelIds: requiredLocalModelIds, + } + : null, + }; +} diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 39b147368..26d38a074 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -20,6 +20,7 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; +import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js"; export type AgentMcpTransportFactory = () => Promise; type VoiceMcpStdioConfig = { @@ -76,6 +77,10 @@ export class VoiceAssistantWebSocketServer { private readonly dictation: { finalTimeoutMs?: number; stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + }; } | null; private readonly voice: { voiceLlmProvider?: AgentProvider | null; @@ -120,6 +125,10 @@ export class VoiceAssistantWebSocketServer { dictation?: { finalTimeoutMs?: number; stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + }; } ) { this.logger = logger.child({ module: "websocket-server" }); diff --git a/packages/website/public/schemas/paseo.config.v1.json b/packages/website/public/schemas/paseo.config.v1.json index 116daee60..cb853f22c 100644 --- a/packages/website/public/schemas/paseo.config.v1.json +++ b/packages/website/public/schemas/paseo.config.v1.json @@ -86,7 +86,7 @@ }, "additionalProperties": false }, - "sherpaOnnx": { + "local": { "type": "object", "properties": { "modelsDir": { @@ -95,32 +95,6 @@ }, "autoDownload": { "type": "boolean" - }, - "stt": { - "type": "object", - "properties": { - "preset": { - "type": "string", - "minLength": 1 - } - }, - "additionalProperties": false - }, - "tts": { - "type": "object", - "properties": { - "preset": { - "type": "string", - "minLength": 1 - }, - "speakerId": { - "type": "number" - }, - "speed": { - "type": "number" - } - }, - "additionalProperties": false } }, "additionalProperties": false @@ -207,10 +181,7 @@ }, "model": { "type": "string", - "enum": [ - "tts-1", - "tts-1-hd" - ] + "minLength": 1 }, "voice": { "type": "string", diff --git a/packages/website/src/routeTree.gen.ts b/packages/website/src/routeTree.gen.ts index 6c8178098..f02250c52 100644 --- a/packages/website/src/routeTree.gen.ts +++ b/packages/website/src/routeTree.gen.ts @@ -13,6 +13,7 @@ import { Route as DocsRouteImport } from './routes/docs' import { Route as IndexRouteImport } from './routes/index' import { Route as DocsIndexRouteImport } from './routes/docs/index' import { Route as DocsWorktreesRouteImport } from './routes/docs/worktrees' +import { Route as DocsVoiceRouteImport } from './routes/docs/voice' import { Route as DocsSecurityRouteImport } from './routes/docs/security' import { Route as DocsConfigurationRouteImport } from './routes/docs/configuration' import { Route as DocsCliRouteImport } from './routes/docs/cli' @@ -38,6 +39,11 @@ const DocsWorktreesRoute = DocsWorktreesRouteImport.update({ path: '/worktrees', getParentRoute: () => DocsRoute, } as any) +const DocsVoiceRoute = DocsVoiceRouteImport.update({ + id: '/voice', + path: '/voice', + getParentRoute: () => DocsRoute, +} as any) const DocsSecurityRoute = DocsSecurityRouteImport.update({ id: '/security', path: '/security', @@ -66,6 +72,7 @@ export interface FileRoutesByFullPath { '/docs/cli': typeof DocsCliRoute '/docs/configuration': typeof DocsConfigurationRoute '/docs/security': typeof DocsSecurityRoute + '/docs/voice': typeof DocsVoiceRoute '/docs/worktrees': typeof DocsWorktreesRoute '/docs/': typeof DocsIndexRoute } @@ -75,6 +82,7 @@ export interface FileRoutesByTo { '/docs/cli': typeof DocsCliRoute '/docs/configuration': typeof DocsConfigurationRoute '/docs/security': typeof DocsSecurityRoute + '/docs/voice': typeof DocsVoiceRoute '/docs/worktrees': typeof DocsWorktreesRoute '/docs': typeof DocsIndexRoute } @@ -86,6 +94,7 @@ export interface FileRoutesById { '/docs/cli': typeof DocsCliRoute '/docs/configuration': typeof DocsConfigurationRoute '/docs/security': typeof DocsSecurityRoute + '/docs/voice': typeof DocsVoiceRoute '/docs/worktrees': typeof DocsWorktreesRoute '/docs/': typeof DocsIndexRoute } @@ -98,6 +107,7 @@ export interface FileRouteTypes { | '/docs/cli' | '/docs/configuration' | '/docs/security' + | '/docs/voice' | '/docs/worktrees' | '/docs/' fileRoutesByTo: FileRoutesByTo @@ -107,6 +117,7 @@ export interface FileRouteTypes { | '/docs/cli' | '/docs/configuration' | '/docs/security' + | '/docs/voice' | '/docs/worktrees' | '/docs' id: @@ -117,6 +128,7 @@ export interface FileRouteTypes { | '/docs/cli' | '/docs/configuration' | '/docs/security' + | '/docs/voice' | '/docs/worktrees' | '/docs/' fileRoutesById: FileRoutesById @@ -156,6 +168,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof DocsWorktreesRouteImport parentRoute: typeof DocsRoute } + '/docs/voice': { + id: '/docs/voice' + path: '/voice' + fullPath: '/docs/voice' + preLoaderRoute: typeof DocsVoiceRouteImport + parentRoute: typeof DocsRoute + } '/docs/security': { id: '/docs/security' path: '/security' @@ -192,6 +211,7 @@ interface DocsRouteChildren { DocsCliRoute: typeof DocsCliRoute DocsConfigurationRoute: typeof DocsConfigurationRoute DocsSecurityRoute: typeof DocsSecurityRoute + DocsVoiceRoute: typeof DocsVoiceRoute DocsWorktreesRoute: typeof DocsWorktreesRoute DocsIndexRoute: typeof DocsIndexRoute } @@ -201,6 +221,7 @@ const DocsRouteChildren: DocsRouteChildren = { DocsCliRoute: DocsCliRoute, DocsConfigurationRoute: DocsConfigurationRoute, DocsSecurityRoute: DocsSecurityRoute, + DocsVoiceRoute: DocsVoiceRoute, DocsWorktreesRoute: DocsWorktreesRoute, DocsIndexRoute: DocsIndexRoute, } diff --git a/packages/website/src/routes/docs.tsx b/packages/website/src/routes/docs.tsx index d393104b2..dfd820c29 100644 --- a/packages/website/src/routes/docs.tsx +++ b/packages/website/src/routes/docs.tsx @@ -7,6 +7,7 @@ export const Route = createFileRoute('/docs')({ const navigation = [ { name: 'Getting started', href: '/docs' }, + { name: 'Voice', href: '/docs/voice' }, { name: 'Git worktrees', href: '/docs/worktrees' }, { name: 'CLI', href: '/docs/cli' }, { name: 'Configuration', href: '/docs/configuration' }, diff --git a/packages/website/src/routes/docs/configuration.tsx b/packages/website/src/routes/docs/configuration.tsx index 169287660..d303acfbe 100644 --- a/packages/website/src/routes/docs/configuration.tsx +++ b/packages/website/src/routes/docs/configuration.tsx @@ -81,75 +81,13 @@ function Configuration() {

Voice

- Voice is provider-based per feature: dictation STT, realtime voice STT, and voice TTS can each - use local or openai. - Defaults are local for all three. -

- 
-{`{
-  "version": 1,
-  "features": {
-    "dictation": { "stt": { "provider": "local" } },
-    "voiceMode": {
-      "llm": { "provider": "claude", "model": "haiku" },
-      "stt": { "provider": "local" },
-      "tts": { "provider": "local" }
-    }
-  },
-  "providers": {
-    "sherpaOnnx": {
-      "modelsDir": "~/.paseo/models/sherpa-onnx",
-      "autoDownload": true,
-      "stt": { "preset": "parakeet-tdt-0.6b-v3-int8" },
-      "tts": { "preset": "pocket-tts-onnx-int8" }
-    }
-  }
-}`}
-
-

- Voice LLM orchestration is agents-only. Set{' '} - features.voiceMode.llm.provider to one of{' '} - claude, codex, or{' '} - opencode. If set, startup is strict and fails if unavailable. - If omitted, Paseo picks the first available voice-enabled agent provider. + Voice is configured through features.dictation and{' '} + features.voiceMode, with provider credentials under{' '} + providers.

- Local voice uses ONNX models (sherpa-onnx + PocketTTS). Default presets are - parakeet-tdt-0.6b-v3-int8 for STT and - pocket-tts-onnx-int8 for TTS. -

-

- With local provider enabled and auto-download on (default outside tests), missing model files are - downloaded at daemon startup into $PASEO_HOME/models/sherpa-onnx. - Downloads are only performed for missing files. -

-

- Local ONNX execution is CPU-first by default. It works without GPU requirements, but performance - depends on your machine. -

-

- To force OpenAI for voice/dictation, configure OpenAI providers explicitly and provide - OPENAI_API_KEY. If OpenAI is selected but credentials are - missing, daemon startup fails fast. -

- 
-{`{
-  "version": 1,
-  "features": {
-    "dictation": { "stt": { "provider": "openai" } },
-    "voiceMode": {
-      "stt": { "provider": "openai" },
-      "tts": { "provider": "openai" }
-    }
-  },
-  "providers": {
-    "openai": { "apiKey": "..." }
-  }
-}`}
-
-

- Realtime voice can create/manage coding agents. Use clear prompts and explicit paths, because - those agents can be launched in arbitrary working directories by request. + For voice philosophy, architecture, and complete local/OpenAI setup examples, see{' '} + Voice docs.

@@ -162,8 +100,10 @@ function Configuration() {
  • OPENAI_API_KEY — override OpenAI provider key
  • PASEO_VOICE_LLM_PROVIDER — override voice LLM provider (claude, codex, opencode)
  • PASEO_DICTATION_STT_PROVIDER, PASEO_VOICE_STT_PROVIDER, PASEO_VOICE_TTS_PROVIDER — override voice provider selection (local or openai)
    • -
  • PASEO_SHERPA_ONNX_MODELS_DIR and PASEO_SHERPA_ONNX_AUTO_DOWNLOAD — control local model directory and download behavior
    • -
  • PASEO_SHERPA_STT_PRESET and PASEO_SHERPA_TTS_PRESET — override local STT/TTS model presets
    • +
  • PASEO_LOCAL_MODELS_DIR and PASEO_LOCAL_AUTO_DOWNLOAD — control local model directory and download behavior
    • +
  • PASEO_DICTATION_LOCAL_STT_MODEL — override local dictation STT model
    • +
  • PASEO_VOICE_LOCAL_STT_MODEL, PASEO_VOICE_LOCAL_TTS_MODEL — override local voice STT/TTS models
    • +
  • PASEO_VOICE_LOCAL_TTS_SPEAKER_ID, PASEO_VOICE_LOCAL_TTS_SPEED — optional local voice TTS tuning
    • diff --git a/packages/website/src/routes/docs/index.tsx b/packages/website/src/routes/docs/index.tsx index afd9cbbd9..69dcfe5ec 100644 --- a/packages/website/src/routes/docs/index.tsx +++ b/packages/website/src/routes/docs/index.tsx @@ -87,22 +87,25 @@ function GettingStarted() {

    Voice Setup

    - Voice supports two providers: local (default) and OpenAI. Local uses ONNX models and will - download missing model files automatically at daemon startup. + Paseo includes first-class voice support with a local-first architecture and configurable speech + providers.

    -
    - $ - export OPENAI_API_KEY=your-key-here -

    - Set OPENAI_API_KEY if you want OpenAI speech providers. - For local defaults and model presets, see Configuration. + For architecture, local model behavior, and provider configuration, see the Voice docs page.

    + + Voice docs +

    Next