diff --git a/docs/data-model.md b/docs/data-model.md index 5964515b8..58ec037b0 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -157,8 +157,8 @@ Single file, validated with `PersistedConfigSchema`. providers: Record }, features: { - dictation: { enabled, stt: { provider, model, confidenceThreshold } }, - voiceMode: { enabled, llm, stt, turnDetection, tts: { provider, model, voice, speakerId, speed } } + dictation: { enabled, stt: { provider, model, language, confidenceThreshold } }, + voiceMode: { enabled, llm, stt: { provider, model, language }, turnDetection, tts: { provider, model, voice, speakerId, speed } } }, log: { level, format, diff --git a/packages/server/src/server/agent/stt-manager.test.ts b/packages/server/src/server/agent/stt-manager.test.ts index 4c5234a8e..e3f6fb7d9 100644 --- a/packages/server/src/server/agent/stt-manager.test.ts +++ b/packages/server/src/server/agent/stt-manager.test.ts @@ -3,6 +3,8 @@ import pino from "pino"; import { EventEmitter } from "node:events"; import { STTManager } from "./stt-manager.js"; +import { PersistedConfigSchema } from "../persisted-config.js"; +import { resolveSpeechConfig } from "../speech/speech-config-resolver.js"; import type { SpeechToTextProvider, StreamingTranscriptionSession, @@ -16,9 +18,11 @@ type StreamingOnHandler = Parameters[1]; class FakeStt implements SpeechToTextProvider { public readonly id = "fake"; + public lastLanguage?: string; constructor(private readonly result: TranscriptionResult) {} - createSession(_params: SessionParams): StreamingTranscriptionSession { + createSession(params: SessionParams): StreamingTranscriptionSession { + this.lastLanguage = params.language; const emitter = new EventEmitter(); const result = this.result; let segmentId = "seg-1"; @@ -92,6 +96,116 @@ class SequencedFakeStt implements SpeechToTextProvider { } describe("STTManager", () => { + function resolveVoiceLanguage(params: { env?: NodeJS.ProcessEnv; persisted?: unknown }): string { + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env: params.env ?? ({} as NodeJS.ProcessEnv), + persisted: PersistedConfigSchema.parse(params.persisted ?? {}), + }); + return result.speech.sttLanguages.voice; + } + + async function transcribeWithResolvedVoiceLanguage(params: { + env?: NodeJS.ProcessEnv; + persisted?: unknown; + }): Promise { + const fakeStt = new FakeStt({ text: "hi", isLowConfidence: false }); + const manager = new STTManager("s1", pino({ level: "silent" }), fakeStt, { + language: resolveVoiceLanguage(params), + }); + await manager.transcribe(Buffer.alloc(2), "audio/pcm;rate=24000"); + return fakeStt; + } + + it("defaults to English when no voice language config is set", async () => { + const fakeStt = await transcribeWithResolvedVoiceLanguage({}); + + expect(fakeStt.lastLanguage).toBe("en"); + }); + + it("uses PASEO_VOICE_LANGUAGE over PASEO_DICTATION_LANGUAGE", async () => { + const fakeStt = await transcribeWithResolvedVoiceLanguage({ + env: { + PASEO_VOICE_LANGUAGE: "pt", + PASEO_DICTATION_LANGUAGE: "es", + } as NodeJS.ProcessEnv, + }); + + expect(fakeStt.lastLanguage).toBe("pt"); + }); + + it("uses PASEO_DICTATION_LANGUAGE when PASEO_VOICE_LANGUAGE is unset", async () => { + const fakeStt = await transcribeWithResolvedVoiceLanguage({ + env: { + PASEO_DICTATION_LANGUAGE: "pt", + } as NodeJS.ProcessEnv, + }); + + expect(fakeStt.lastLanguage).toBe("pt"); + }); + + it("treats empty voice language env vars as unset", async () => { + const fakeStt = await transcribeWithResolvedVoiceLanguage({ + env: { + PASEO_VOICE_LANGUAGE: "", + PASEO_DICTATION_LANGUAGE: " ", + } as NodeJS.ProcessEnv, + }); + + expect(fakeStt.lastLanguage).toBe("en"); + }); + + it("uses settings voice STT language when no env var is set", async () => { + const fakeStt = await transcribeWithResolvedVoiceLanguage({ + persisted: { + features: { + voiceMode: { + stt: { + language: "fr", + }, + }, + }, + }, + }); + + expect(fakeStt.lastLanguage).toBe("fr"); + }); + + it("uses env voice language over settings voice STT language", async () => { + const fakeStt = await transcribeWithResolvedVoiceLanguage({ + env: { + PASEO_VOICE_LANGUAGE: "pt", + } as NodeJS.ProcessEnv, + persisted: { + features: { + voiceMode: { + stt: { + language: "fr", + }, + }, + }, + }, + }); + + expect(fakeStt.lastLanguage).toBe("pt"); + }); + + it("falls back to settings dictation STT language when voice language is unset", async () => { + const fakeStt = await transcribeWithResolvedVoiceLanguage({ + persisted: { + features: { + dictation: { + stt: { + language: "es", + }, + }, + }, + }, + }); + + expect(fakeStt.lastLanguage).toBe("es"); + }); + it("returns empty text for low-confidence transcriptions", async () => { const manager = new STTManager( "s1", diff --git a/packages/server/src/server/agent/stt-manager.ts b/packages/server/src/server/agent/stt-manager.ts index 66a32babd..391287f0f 100644 --- a/packages/server/src/server/agent/stt-manager.ts +++ b/packages/server/src/server/agent/stt-manager.ts @@ -111,6 +111,10 @@ export interface SessionTranscriptionResult extends TranscriptionResult { format: string; } +export interface STTManagerOptions { + language?: string; +} + /** * Per-session STT manager * Handles speech-to-text transcription @@ -119,15 +123,18 @@ export class STTManager { private readonly sessionId: string; private readonly logger: pino.Logger; private readonly resolveStt: () => SpeechToTextProvider | null; + private readonly language: string; constructor( sessionId: string, logger: pino.Logger, stt: Resolvable, + options?: STTManagerOptions, ) { this.sessionId = sessionId; this.logger = logger.child({ module: "agent", component: "stt-manager", sessionId }); this.resolveStt = toResolver(stt); + this.language = options?.language ?? "en"; } public getProvider(): SpeechToTextProvider | null { @@ -171,7 +178,7 @@ export class STTManager { const session = stt.createSession({ logger: this.logger.child({ component: "stt-session" }), - language: "en", + language: this.language, }); const pcmForModel = preparePcmForModel(audio, format, session.requiredSampleRate); diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index e0713cc1b..22baa93b1 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -195,8 +195,14 @@ function summarizeAgentMcpDebugBody(body: unknown): Record { export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig; export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig; +export interface PaseoSpeechSttLanguages { + dictation: string; + voice: string; +} + export interface PaseoSpeechConfig { providers: RequestedSpeechProviders; + sttLanguages?: PaseoSpeechSttLanguages; local?: PaseoLocalSpeechConfig; } diff --git a/packages/server/src/server/dictation/dictation-stream-manager.test.ts b/packages/server/src/server/dictation/dictation-stream-manager.test.ts index 1f53317d1..bcba41b52 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.test.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.test.ts @@ -3,6 +3,8 @@ import { EventEmitter } from "node:events"; import pino from "pino"; import { DictationStreamManager } from "./dictation-stream-manager.js"; +import { PersistedConfigSchema } from "../persisted-config.js"; +import { resolveSpeechConfig } from "../speech/speech-config-resolver.js"; import type { SpeechToTextProvider, StreamingTranscriptionSession, @@ -51,10 +53,12 @@ class FakeRealtimeSession extends EventEmitter implements StreamingTranscription class FakeSttProvider implements SpeechToTextProvider { public readonly id = "fake"; + public lastLanguage?: string; constructor(private readonly session: FakeRealtimeSession) {} createSession( - _params: Parameters[0], + params: Parameters[0], ): StreamingTranscriptionSession { + this.lastLanguage = params.language; return this.session; } } @@ -123,6 +127,97 @@ describe("DictationStreamManager (finish buffer-too-small tolerance)", () => { }); describe("DictationStreamManager (provider-agnostic provider)", () => { + function resolveDictationLanguage(params: { + env?: NodeJS.ProcessEnv; + persisted?: unknown; + }): string { + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env: params.env ?? ({} as NodeJS.ProcessEnv), + persisted: PersistedConfigSchema.parse(params.persisted ?? {}), + }); + return result.speech.sttLanguages.dictation; + } + + async function startWithResolvedDictationLanguage(params: { + env?: NodeJS.ProcessEnv; + persisted?: unknown; + }): Promise { + const session = new FakeRealtimeSession(); + const sttProvider = new FakeSttProvider(session); + const manager = new DictationStreamManager({ + logger: pino({ level: "silent" }), + emit: () => {}, + sessionId: "s1", + stt: sttProvider, + language: resolveDictationLanguage(params), + }); + + await manager.handleStart("d-lang", "audio/pcm;rate=24000;bits=16"); + return sttProvider; + } + + it("defaults to English when dictation language config is unset", async () => { + const sttProvider = await startWithResolvedDictationLanguage({}); + + expect(sttProvider.lastLanguage).toBe("en"); + }); + + it("uses PASEO_DICTATION_LANGUAGE when set", async () => { + const sttProvider = await startWithResolvedDictationLanguage({ + env: { + PASEO_DICTATION_LANGUAGE: "pt", + } as NodeJS.ProcessEnv, + }); + + expect(sttProvider.lastLanguage).toBe("pt"); + }); + + it("treats empty PASEO_DICTATION_LANGUAGE as unset", async () => { + const sttProvider = await startWithResolvedDictationLanguage({ + env: { + PASEO_DICTATION_LANGUAGE: " ", + } as NodeJS.ProcessEnv, + }); + + expect(sttProvider.lastLanguage).toBe("en"); + }); + + it("uses settings dictation STT language when env var is unset", async () => { + const sttProvider = await startWithResolvedDictationLanguage({ + persisted: { + features: { + dictation: { + stt: { + language: "fr", + }, + }, + }, + }, + }); + + expect(sttProvider.lastLanguage).toBe("fr"); + }); + + it("uses env dictation language over settings dictation STT language", async () => { + const sttProvider = await startWithResolvedDictationLanguage({ + env: { + PASEO_DICTATION_LANGUAGE: "pt", + } as NodeJS.ProcessEnv, + persisted: { + features: { + dictation: { + stt: { + language: "fr", + }, + }, + }, + }, + }); + + expect(sttProvider.lastLanguage).toBe("pt"); + }); + it("does not require OPENAI_API_KEY", async () => { const original = process.env.OPENAI_API_KEY; delete process.env.OPENAI_API_KEY; diff --git a/packages/server/src/server/dictation/dictation-stream-manager.ts b/packages/server/src/server/dictation/dictation-stream-manager.ts index c87047ea1..db3cab8ae 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.ts @@ -130,6 +130,7 @@ export class DictationStreamManager { private readonly emit: (msg: DictationStreamOutboundMessage) => void; private readonly sessionId: string; private readonly resolveStt: () => SpeechToTextProvider | null; + private readonly language: string; private readonly finalTimeoutMs: number; private readonly autoCommitSeconds: number; private readonly streams = new Map(); @@ -139,6 +140,7 @@ export class DictationStreamManager { emit: (msg: DictationStreamOutboundMessage) => void; sessionId: string; stt: Resolvable; + language?: string; finalTimeoutMs?: number; autoCommitSeconds?: number; }) { @@ -146,6 +148,7 @@ export class DictationStreamManager { this.emit = params.emit; this.sessionId = params.sessionId; this.resolveStt = toResolver(params.stt); + this.language = params.language ?? "en"; this.finalTimeoutMs = params.finalTimeoutMs ?? DEFAULT_DICTATION_FINAL_TIMEOUT_MS; this.autoCommitSeconds = params.autoCommitSeconds ?? @@ -176,7 +179,7 @@ export class DictationStreamManager { try { stt = sttProvider.createSession({ logger: this.logger.child({ dictationId }), - language: "en", + language: this.language, prompt: transcriptionPrompt, }); } catch (error) { diff --git a/packages/server/src/server/persisted-config.test.ts b/packages/server/src/server/persisted-config.test.ts index bcd277eba..a17d5129f 100644 --- a/packages/server/src/server/persisted-config.test.ts +++ b/packages/server/src/server/persisted-config.test.ts @@ -483,6 +483,26 @@ describe("PersistedConfigSchema voice mode config", () => { expect(parsed.features?.voiceMode?.turnDetection?.provider).toBe("local"); }); + + test("accepts trimmed STT language fields", () => { + const parsed = PersistedConfigSchema.parse({ + features: { + dictation: { + stt: { + language: " fr ", + }, + }, + voiceMode: { + stt: { + language: " de ", + }, + }, + }, + }); + + expect(parsed.features?.dictation?.stt?.language).toBe("fr"); + expect(parsed.features?.voiceMode?.stt?.language).toBe("de"); + }); }); describe.skipIf(process.platform === "win32")("persisted config file permissions", () => { diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 5e8f4386e..af8b6e132 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -86,6 +86,7 @@ const FeatureDictationSchema = z .object({ provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), + language: z.string().trim().min(1).optional(), confidenceThreshold: z.number().optional(), }) .strict() @@ -107,6 +108,7 @@ const FeatureVoiceModeSchema = z .object({ provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), + language: z.string().trim().min(1).optional(), }) .strict() .optional(), diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 37c53a1ef..41ab862b7 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -546,6 +546,7 @@ export interface SessionOptions { daemonConfigStore: DaemonConfigStore; mcpBaseUrl?: string | null; stt: Resolvable; + sttLanguage?: string; tts: Resolvable; terminalManager: TerminalManager | null; providerSnapshotManager?: ProviderSnapshotManager; @@ -572,6 +573,7 @@ export interface SessionOptions { dictation?: { finalTimeoutMs?: number; stt?: Resolvable; + sttLanguage?: string; getSpeechReadiness?: () => SpeechReadinessSnapshot; }; agentProviderRuntimeSettings?: AgentProviderRuntimeSettingsMap; @@ -783,6 +785,7 @@ export class Session { private registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void; private unregisterVoiceCallerContext?: (agentId: string) => void; private getSpeechReadiness?: () => SpeechReadinessSnapshot; + private readonly sttLanguage: string; private readonly agentProviderRuntimeSettings: AgentProviderRuntimeSettingsMap | undefined; private readonly providerOverrides: Record | undefined; private readonly isDev: boolean; @@ -814,6 +817,7 @@ export class Session { daemonConfigStore, mcpBaseUrl, stt, + sttLanguage, tts, terminalManager, providerSnapshotManager, @@ -875,6 +879,7 @@ export class Session { this.getDaemonTcpPort = getDaemonTcpPort ?? null; this.getDaemonTcpHost = getDaemonTcpHost ?? null; this.resolveScriptHealth = resolveScriptHealth ?? null; + this.sttLanguage = sttLanguage ?? "en"; this.subscribeToOptionalManagers(); this.bindVoiceBridges({ voice, voiceBridge, dictation }); this.agentProviderRuntimeSettings = agentProviderRuntimeSettings; @@ -890,7 +895,7 @@ export class Session { buildWorkspaceDescriptor: (input) => this.buildWorkspaceDescriptor(input), }); - this.initializePerSessionManagers({ tts, stt, dictation }); + this.initializePerSessionManagers({ tts, stt, sttLanguage, dictation }); // Initialize agent MCP client asynchronously void this.initializeAgentMcp(); @@ -1191,16 +1196,20 @@ export class Session { private initializePerSessionManagers(params: { tts: SessionOptions["tts"]; stt: SessionOptions["stt"]; + sttLanguage: SessionOptions["sttLanguage"]; dictation: SessionOptions["dictation"]; }): void { - const { tts, stt, dictation } = params; + const { tts, stt, sttLanguage, dictation } = params; this.ttsManager = new TTSManager(this.sessionId, this.sessionLogger, tts); - this.sttManager = new STTManager(this.sessionId, this.sessionLogger, stt); + this.sttManager = new STTManager(this.sessionId, this.sessionLogger, stt, { + language: sttLanguage, + }); this.dictationStreamManager = new DictationStreamManager({ logger: this.sessionLogger, sessionId: this.sessionId, emit: (msg) => this.handleDictationManagerMessage(msg), stt: dictation?.stt ?? null, + language: dictation?.sttLanguage, finalTimeoutMs: dictation?.finalTimeoutMs, }); } @@ -2765,6 +2774,7 @@ export class Session { logger: this.sessionLogger.child({ component: "voice-turn-controller" }), turnDetection, stt, + sttLanguage: this.sttLanguage, callbacks: { onSpeechStarted: async () => { this.sessionLogger.debug("Voice VAD speech_started"); diff --git a/packages/server/src/server/speech/providers/local/config.ts b/packages/server/src/server/speech/providers/local/config.ts index fbefe2d3d..b4bf0d483 100644 --- a/packages/server/src/server/speech/providers/local/config.ts +++ b/packages/server/src/server/speech/providers/local/config.ts @@ -29,13 +29,21 @@ export interface LocalSpeechProviderConfig { export interface ResolvedLocalSpeechConfig { local: LocalSpeechProviderConfig | undefined; + sttLanguages: LocalSpeechSttLanguageConfig; } export type { LocalSpeechModelId, LocalSttModelId, LocalTtsModelId }; const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech"); +const DEFAULT_STT_LANGUAGE = "en"; + +export interface LocalSpeechSttLanguageConfig { + dictation: string; + voice: string; +} const NumberLikeSchema = z.union([z.number(), z.string().trim().min(1)]); +const LanguageSchema = z.string().trim().min(1).default(DEFAULT_STT_LANGUAGE); const OptionalFiniteNumberSchema = NumberLikeSchema.pipe(z.coerce.number().finite()).optional(); @@ -47,6 +55,8 @@ const LocalSpeechResolutionSchema = z.object({ dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL), + dictationLanguage: LanguageSchema, + voiceLanguage: LanguageSchema, voiceLocalTtsSpeakerId: OptionalIntegerSchema, voiceLocalTtsSpeed: OptionalFiniteNumberSchema, }); @@ -90,6 +100,37 @@ function firstDefinedValue(values: Array): T | undefine return undefined; } +function firstNonEmptyString(values: Array): string | undefined { + for (const value of values) { + const trimmed = value?.trim(); + if (trimmed) { + return trimmed; + } + } + return undefined; +} + +function buildLocalSpeechLanguageResolutionInput(params: { + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; +}): Record { + const { env, persisted } = params; + return { + dictationLanguage: firstNonEmptyString([ + env.PASEO_DICTATION_LANGUAGE, + persisted.features?.dictation?.stt?.language, + DEFAULT_STT_LANGUAGE, + ]), + voiceLanguage: firstNonEmptyString([ + env.PASEO_VOICE_LANGUAGE, + env.PASEO_DICTATION_LANGUAGE, + persisted.features?.voiceMode?.stt?.language, + persisted.features?.dictation?.stt?.language, + DEFAULT_STT_LANGUAGE, + ]), + }; +} + function buildLocalSpeechResolutionInput(params: { paseoHome: string; env: NodeJS.ProcessEnv; @@ -132,6 +173,7 @@ function buildLocalSpeechResolutionInput(params: { ), DEFAULT_LOCAL_TTS_MODEL, ]), + ...buildLocalSpeechLanguageResolutionInput({ env, persisted }), voiceLocalTtsSpeakerId: firstDefinedValue([ env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID, persisted.features?.voiceMode?.tts?.speakerId, @@ -159,6 +201,10 @@ export function resolveLocalSpeechConfig(params: { (parsed.voiceLocalTtsModel === "kokoro-en-v0_19" ? 0 : undefined); return { + sttLanguages: { + dictation: parsed.dictationLanguage, + voice: parsed.voiceLanguage, + }, local: parsed.includeProviderConfig ? { modelsDir: parsed.modelsDir, diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts index 37e48f5ea..86caed8cc 100644 --- a/packages/server/src/server/speech/speech-config-resolver.test.ts +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -51,6 +51,10 @@ describe("resolveSpeechConfig", () => { expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8"); expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19"); expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0); + expect(result.speech.sttLanguages).toEqual({ + dictation: "en", + voice: "en", + }); }); test("resolves feature-scoped local model env vars", () => { @@ -71,6 +75,8 @@ describe("resolveSpeechConfig", () => { PASEO_VOICE_LOCAL_TTS_MODEL: "kitten", PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5", PASEO_VOICE_LOCAL_TTS_SPEED: "1.35", + PASEO_DICTATION_LANGUAGE: "es", + PASEO_VOICE_LANGUAGE: "pt", PASEO_LOCAL_MODELS_DIR: "/tmp/models", OPENAI_API_KEY: "env-key", PASEO_VOICE_STT_PROVIDER: "openai", @@ -119,10 +125,45 @@ describe("resolveSpeechConfig", () => { expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16"); expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5); expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35); + expect(result.speech.sttLanguages).toEqual({ + dictation: "es", + voice: "pt", + }); expect(result.openai?.apiKey).toBe("env-key"); expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe"); }); + test("resolves STT language from env, settings, and voice-to-dictation fallback", () => { + const persisted = PersistedConfigSchema.parse({ + features: { + dictation: { + stt: { + language: "fr", + }, + }, + voiceMode: { + stt: { + language: "de", + }, + }, + }, + }); + + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env: { + PASEO_DICTATION_LANGUAGE: "es", + PASEO_VOICE_LANGUAGE: " ", + } as NodeJS.ProcessEnv, + persisted, + }); + + expect(result.speech.sttLanguages).toEqual({ + dictation: "es", + voice: "es", + }); + }); + test("ignores deprecated shared local model env vars", () => { const persisted = PersistedConfigSchema.parse({}); const env = { diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts index c2a0c27f9..95c78707a 100644 --- a/packages/server/src/server/speech/speech-config-resolver.ts +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -173,6 +173,7 @@ export function resolveSpeechConfig(params: { openai, speech: { providers, + sttLanguages: local.sttLanguages, ...(local.local ? { local: local.local } : {}), }, }; diff --git a/packages/server/src/server/speech/speech-runtime.test.ts b/packages/server/src/server/speech/speech-runtime.test.ts index a891e7656..f368d5977 100644 --- a/packages/server/src/server/speech/speech-runtime.test.ts +++ b/packages/server/src/server/speech/speech-runtime.test.ts @@ -67,7 +67,13 @@ function createStubTurnDetection(id: string): TurnDetectionProvider { } function createSpeechConfig(providers: PaseoSpeechConfig["providers"]): PaseoSpeechConfig { - return { providers }; + return { + providers, + sttLanguages: { + dictation: "en", + voice: "en", + }, + }; } describe("createSpeechService readiness", () => { diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts index 5a9236ce1..8da70afd8 100644 --- a/packages/server/src/server/speech/speech-runtime.ts +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -342,9 +342,11 @@ function resolveEffectiveProviderIds(params: { export interface SpeechService { resolveStt: () => SpeechToTextProvider | null; + resolveSttLanguage: () => string; resolveTts: () => TextToSpeechProvider | null; resolveTurnDetection: () => TurnDetectionProvider | null; resolveDictationStt: () => SpeechToTextProvider | null; + resolveDictationSttLanguage: () => string; getReadiness: () => SpeechReadinessSnapshot; onReadinessChange: (listener: (snapshot: SpeechReadinessSnapshot) => void) => () => void; start: () => void; @@ -709,8 +711,10 @@ export function createSpeechService(params: { return { resolveTurnDetection: () => turnDetectionService, resolveStt: () => sttService, + resolveSttLanguage: () => speechConfig?.sttLanguages?.voice ?? "en", resolveTts: () => ttsService, resolveDictationStt: () => dictationSttService, + resolveDictationSttLanguage: () => speechConfig?.sttLanguages?.dictation ?? "en", getReadiness: () => lastPublishedReadinessSnapshot ?? computeReadinessSnapshot(), onReadinessChange: subscribeSpeechReadiness, start, diff --git a/packages/server/src/server/voice/voice-turn-controller.test.ts b/packages/server/src/server/voice/voice-turn-controller.test.ts index bc4ab0d3d..6f2a304c5 100644 --- a/packages/server/src/server/voice/voice-turn-controller.test.ts +++ b/packages/server/src/server/voice/voice-turn-controller.test.ts @@ -81,10 +81,14 @@ function createFakeTurnDetectionProvider(session: FakeTurnDetectionSession): Tur }; } -function createFakeSttProvider(sessions: FakeSttSession[]): SpeechToTextProvider { +function createFakeSttProvider( + sessions: FakeSttSession[], + captureLanguage?: (language: string | undefined) => void, +): SpeechToTextProvider { return { id: "local", - createSession() { + createSession(params) { + captureLanguage?.(params.language); const session = new FakeSttSession(); sessions.push(session); return session; @@ -98,10 +102,13 @@ async function settleSerialQueue(): Promise { await Promise.resolve(); } -function createControllerHarness() { +function createControllerHarness(options?: { sttLanguage?: string }) { const detector = new FakeTurnDetectionSession(); const sttSessions: FakeSttSession[] = []; - const stt = createFakeSttProvider(sttSessions); + let lastSttLanguage: string | undefined; + const stt = createFakeSttProvider(sttSessions, (language) => { + lastSttLanguage = language; + }); const onSpeechStarted = vi.fn(async () => {}); const onSpeechStopped = vi.fn(async () => {}); const onPartialTranscript = vi.fn( @@ -123,6 +130,7 @@ function createControllerHarness() { logger: pino({ level: "silent" }), turnDetection: createFakeTurnDetectionProvider(detector), stt, + sttLanguage: options?.sttLanguage, callbacks: { onSpeechStarted, onSpeechStopped, @@ -136,6 +144,7 @@ function createControllerHarness() { controller, detector, sttSessions, + getLastSttLanguage: () => lastSttLanguage, onSpeechStarted, onSpeechStopped, onPartialTranscript, @@ -145,6 +154,16 @@ function createControllerHarness() { } describe("voice turn controller", () => { + it("passes configured language to streaming STT", async () => { + const harness = createControllerHarness({ sttLanguage: "pt" }); + + await harness.controller.start(); + harness.detector.emit("speech_started"); + await settleSerialQueue(); + + expect(harness.getLastSttLanguage()).toBe("pt"); + }); + it("forwards audio to the detector and streaming STT without submitting buffered utterances", async () => { const harness = createControllerHarness(); diff --git a/packages/server/src/server/voice/voice-turn-controller.ts b/packages/server/src/server/voice/voice-turn-controller.ts index 2cab7f2ba..eb7e1d3dd 100644 --- a/packages/server/src/server/voice/voice-turn-controller.ts +++ b/packages/server/src/server/voice/voice-turn-controller.ts @@ -94,6 +94,7 @@ export function createVoiceTurnController(params: { logger: Logger; turnDetection: TurnDetectionProvider; stt: SpeechToTextProvider; + sttLanguage?: string; callbacks: VoiceTurnControllerCallbacks; }): VoiceTurnController { const detector = params.turnDetection.createSession({ @@ -319,7 +320,7 @@ export function createVoiceTurnController(params: { function createSttSession(): StreamingTranscriptionSession { const session = params.stt.createSession({ logger: params.logger.child({ component: "stt" }), - language: "en", + language: params.sttLanguage ?? "en", }); session.on("transcript", handleSttTranscript); session.on("committed", ({ segmentId }) => { diff --git a/packages/server/src/server/websocket-server.relay-reconnect.test.ts b/packages/server/src/server/websocket-server.relay-reconnect.test.ts index 31119a340..0a44802f7 100644 --- a/packages/server/src/server/websocket-server.relay-reconnect.test.ts +++ b/packages/server/src/server/websocket-server.relay-reconnect.test.ts @@ -215,8 +215,17 @@ function createServer(options?: { speechReadiness?: SpeechReadinessSnapshot | nu undefined, speechReadiness ? { + resolveStt: () => null, + resolveSttLanguage: () => "en", + resolveTts: () => null, + resolveTurnDetection: () => null, + resolveDictationStt: () => null, + resolveDictationSttLanguage: () => "en", getReadiness: () => speechReadiness, onReadinessChange: vi.fn(() => () => {}), + start: vi.fn(), + stop: vi.fn(), + ready: Promise.resolve(), } : undefined, undefined, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 4f63c0fed..ac240a2bd 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -878,6 +878,7 @@ export class VoiceAssistantWebSocketServer { daemonConfigStore: this.daemonConfigStore, mcpBaseUrl: this.mcpBaseUrl, stt: () => this.speech?.resolveStt() ?? null, + sttLanguage: this.speech?.resolveSttLanguage() ?? "en", tts: () => this.speech?.resolveTts() ?? null, terminalManager: this.terminalManager, providerSnapshotManager: this.providerSnapshotManager, @@ -910,6 +911,7 @@ export class VoiceAssistantWebSocketServer { ? { finalTimeoutMs: this.dictation?.finalTimeoutMs, stt: () => this.speech?.resolveDictationStt() ?? null, + sttLanguage: this.speech?.resolveDictationSttLanguage() ?? "en", getSpeechReadiness: () => this.speech!.getReadiness(), } : undefined, diff --git a/public-docs/configuration.md b/public-docs/configuration.md index 0d2b4766a..ae68c9702 100644 --- a/public-docs/configuration.md +++ b/public-docs/configuration.md @@ -161,6 +161,7 @@ In the mobile app, enter the password in the direct connection setup screen. - `PASEO_LOCAL_MODELS_DIR`, control local model directory - `PASEO_DICTATION_LOCAL_STT_MODEL`, override local dictation STT model - `PASEO_VOICE_LOCAL_STT_MODEL`, `PASEO_VOICE_LOCAL_TTS_MODEL`, override local voice STT/TTS models +- `PASEO_DICTATION_LANGUAGE`, `PASEO_VOICE_LANGUAGE`, override dictation and voice STT language - `PASEO_VOICE_LOCAL_TTS_SPEAKER_ID`, `PASEO_VOICE_LOCAL_TTS_SPEED`, optional local voice TTS tuning ## Schema diff --git a/public-docs/voice.md b/public-docs/voice.md index 5d5dafe7a..a8f56a71a 100644 --- a/public-docs/voice.md +++ b/public-docs/voice.md @@ -24,7 +24,7 @@ This keeps credentials and execution in your environment and avoids introducing ## Local Speech -Local speech defaults to model IDs `parakeet-tdt-0.6b-v3-int8` (STT) and `kokoro-en-v0_19` (TTS, speaker 0 / voice 00). +Local speech defaults to model IDs `parakeet-tdt-0.6b-v3-int8` (STT) and `kokoro-en-v0_19` (TTS, speaker 0 / voice 00). STT language defaults to `en`. Missing models are downloaded at daemon startup into `$PASEO_HOME/models/local-speech`. Downloads happen only for missing files. @@ -32,10 +32,12 @@ Missing models are downloaded at daemon startup into `$PASEO_HOME/models/local-s { "version": 1, "features": { - "dictation": { "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8" } }, + "dictation": { + "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8", "language": "en" } + }, "voiceMode": { "llm": { "provider": "claude", "model": "haiku" }, - "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8" }, + "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8", "language": "en" }, "tts": { "provider": "local", "model": "kokoro-en-v0_19", "speakerId": 0 } } }, @@ -47,6 +49,8 @@ Missing models are downloaded at daemon startup into `$PASEO_HOME/models/local-s } ``` +Set `features.dictation.stt.language` for dictation and `features.voiceMode.stt.language` for realtime voice. If voice language is omitted, Paseo uses the dictation language before falling back to `en`. + ## OpenAI Speech Option You can switch dictation, voice STT, and voice TTS to OpenAI by setting provider fields to `openai` and providing `OPENAI_API_KEY`. @@ -74,6 +78,8 @@ You can switch dictation, voice STT, and voice TTS to OpenAI by setting provider - `PASEO_LOCAL_MODELS_DIR`, local model storage directory - `PASEO_DICTATION_LOCAL_STT_MODEL`, local dictation STT model ID - `PASEO_VOICE_LOCAL_STT_MODEL`, `PASEO_VOICE_LOCAL_TTS_MODEL`, local voice STT/TTS model IDs +- `PASEO_DICTATION_LANGUAGE`, dictation STT language +- `PASEO_VOICE_LANGUAGE`, realtime voice STT language; falls back to `PASEO_DICTATION_LANGUAGE` when unset - `PASEO_VOICE_LOCAL_TTS_SPEAKER_ID`, `PASEO_VOICE_LOCAL_TTS_SPEED`, optional local voice TTS tuning ## Operational Notes