jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Update files

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-06 23:49:40 +07:00
parent 4a4cff105f
commit 9bd26a46e1
23 changed files with 181 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/scripts/download-speech-models.ts
+13 -9
View File
@@ -1,12 +1,16 @@
import { resolvePaseoHome } from "../src/server/paseo-home.js";
import { createRootLogger } from "../src/server/logger.js";
import { ensureSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-downloader.js";
import type { SherpaOnnxModelId } from "../src/server/speech/providers/local/sherpa/model-catalog.js";
import {
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
ensureLocalSpeechModels,
type LocalSpeechModelId,
} from "../src/server/speech/providers/local/models.js";
function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxModelId[] } {
function parseArgs(argv: string[]): { modelsDir: string; modelIds: LocalSpeechModelId[] } {
const home = resolvePaseoHome();
let modelsDir = process.env.PASEO_SHERPA_ONNX_MODELS_DIR || `${home}/models/sherpa-onnx`;
const modelIds: SherpaOnnxModelId[] = [];
let modelsDir = process.env.PASEO_LOCAL_MODELS_DIR || `${home}/models/local-speech`;
const modelIds: LocalSpeechModelId[] = [];
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
@@ -16,7 +20,7 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod
continue;
}
if (arg === "--model") {
const id = argv[i + 1] as SherpaOnnxModelId | undefined;
const id = argv[i + 1] as LocalSpeechModelId | undefined;
if (!id) {
throw new Error("--model requires a value");
}
@@ -27,8 +31,8 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod
}
if (modelIds.length === 0) {
const stt = (process.env.PASEO_SHERPA_STT_PRESET || "zipformer-bilingual-zh-en-2023-02-20") as SherpaOnnxModelId;
const tts = (process.env.PASEO_SHERPA_TTS_PRESET || "pocket-tts-onnx-int8") as SherpaOnnxModelId;
const stt = (process.env.PASEO_LOCAL_STT_MODEL || DEFAULT_LOCAL_STT_MODEL) as LocalSpeechModelId;
const tts = (process.env.PASEO_LOCAL_TTS_MODEL || DEFAULT_LOCAL_TTS_MODEL) as LocalSpeechModelId;
modelIds.push(stt, tts);
}
@@ -38,5 +42,5 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod
const logger = createRootLogger({ level: "info", format: "pretty" });
const { modelsDir, modelIds } = parseArgs(process.argv.slice(2));
await ensureSherpaOnnxModels({ modelsDir, modelIds, autoDownload: true, logger });
await ensureLocalSpeechModels({ modelsDir, modelIds, autoDownload: true, logger });
logger.info({ modelsDir, modelIds }, "Done downloading speech models");
packages/server/scripts/list-speech-models.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { listSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-catalog.js";
import { listLocalSpeechModels } from "../src/server/speech/providers/local/models.js";
const models = listSherpaOnnxModels()
const models = listLocalSpeechModels()
.slice()
.sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id));
packages/server/src/server/bootstrap.smoke.test.ts
-10
View File
@@ -18,11 +18,6 @@ describe("paseo daemon bootstrap", () => {
voiceStt: { provider: "openai", explicit: true },
voiceTts: { provider: "openai", explicit: true },
},
localModels: {
dictationStt: "parakeet-tdt-0.6b-v3-int8",
voiceStt: "parakeet-tdt-0.6b-v3-int8",
voiceTts: "pocket-tts-onnx-int8",
},
},
});
try {
@@ -68,11 +63,6 @@ describe("paseo daemon bootstrap", () => {
voiceStt: { provider: "openai", explicit: true },
voiceTts: { provider: "openai", explicit: true },
},
localModels: {
dictationStt: "parakeet-tdt-0.6b-v3-int8",
voiceStt: "parakeet-tdt-0.6b-v3-int8",
voiceTts: "pocket-tts-onnx-int8",
},
},
};
packages/server/src/server/bootstrap.ts
-7
View File
@@ -108,13 +108,6 @@ export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig;
export type PaseoSpeechConfig = {
providers: RequestedSpeechProviders;
local?: PaseoLocalSpeechConfig;
localModels: {
dictationStt: string;
voiceStt: string;
voiceTts: string;
voiceTtsSpeakerId?: number;
voiceTtsSpeed?: number;
};
};
export type PaseoDaemonConfig = {
packages/server/src/server/config.ts
+12 -14
View File
@@ -1,9 +1,10 @@
import path from "node:path";
import { z } from "zod";
import type { PaseoDaemonConfig } from "./bootstrap.js";
import { loadPersistedConfig } from "./persisted-config.js";
import type { AgentProvider } from "./agent/agent-sdk-types.js";
import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
import { AgentProviderSchema } from "./agent/provider-manifest.js";
import { resolveSpeechConfig } from "./speech/speech-config-resolver.js";
import {
mergeAllowedHosts,
@@ -26,17 +27,14 @@ export type CliConfigOverrides = Partial<{
allowedHosts: AllowedHostsConfig;
}>;
function parseVoiceLlmProviderId(value: unknown): AgentProvider | null {
if (typeof value !== "string") {
return null;
}
const normalized = value.trim().toLowerCase();
if (!normalized) {
return null;
}
return (AGENT_PROVIDER_IDS as readonly string[]).includes(normalized)
? (normalized as AgentProvider)
: null;
const OptionalVoiceLlmProviderSchema = z
.union([z.string(), z.null(), z.undefined()])
.transform((value): string | null => (typeof value === "string" ? value.trim().toLowerCase() : null))
.pipe(z.union([AgentProviderSchema, z.null()]));
function parseOptionalVoiceLlmProvider(value: unknown): AgentProvider | null {
const parsed = OptionalVoiceLlmProviderSchema.safeParse(value);
return parsed.success ? parsed.data : null;
}
export function loadConfig(
@@ -97,8 +95,8 @@ export function loadConfig(
persisted,
});
const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER);
const persistedVoiceLlmProvider = parseVoiceLlmProviderId(
const envVoiceLlmProvider = parseOptionalVoiceLlmProvider(env.PASEO_VOICE_LLM_PROVIDER);
const persistedVoiceLlmProvider = parseOptionalVoiceLlmProvider(
persisted.features?.voiceMode?.llm?.provider
);
const voiceLlmProvider = envVoiceLlmProvider ?? persistedVoiceLlmProvider ?? null;
packages/server/src/server/daemon-client.e2e.test.ts
+10 -15
View File
@@ -101,11 +101,6 @@ describe("daemon client E2E", () => {
voiceStt: { provider: "openai" as const, explicit: true },
voiceTts: { provider: "openai" as const, explicit: true },
},
localModels: {
dictationStt: "parakeet-tdt-0.6b-v3-int8",
voiceStt: "parakeet-tdt-0.6b-v3-int8",
voiceTts: "pocket-tts-onnx-int8",
},
}
: hasLocalSpeech
? {
@@ -116,16 +111,16 @@ describe("daemon client E2E", () => {
},
local: {
modelsDir: localModelsDir,
},
localModels: {
dictationStt:
process.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
"zipformer-bilingual-zh-en-2023-02-20",
voiceStt:
process.env.PASEO_VOICE_LOCAL_STT_MODEL ??
"zipformer-bilingual-zh-en-2023-02-20",
voiceTts:
process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
models: {
dictationStt:
process.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
"zipformer-bilingual-zh-en-2023-02-20",
voiceStt:
process.env.PASEO_VOICE_LOCAL_STT_MODEL ??
"zipformer-bilingual-zh-en-2023-02-20",
voiceTts:
process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
},
},
}
: undefined;
packages/server/src/server/session.ts
+13 -15
View File
@@ -103,13 +103,11 @@ import {
import { getProjectIcon } from "../utils/project-icon.js";
import { expandTilde } from "../utils/path.js";
import {
ensureSherpaOnnxModels,
getSherpaOnnxModelDir,
} from "./speech/providers/local/sherpa/model-downloader.js";
import {
listSherpaOnnxModels,
type SherpaOnnxModelId,
} from "./speech/providers/local/sherpa/model-catalog.js";
ensureLocalSpeechModels,
getLocalSpeechModelDir,
listLocalSpeechModels,
type LocalSpeechModelId,
} from "./speech/providers/local/models.js";
import type pino from "pino";
const execAsync = promisify(exec);
@@ -222,7 +220,7 @@ export type SessionOptions = {
stt?: SpeechToTextProvider | null;
localModels?: {
modelsDir: string;
defaultModelIds: SherpaOnnxModelId[];
defaultModelIds: LocalSpeechModelId[];
};
};
};
@@ -371,7 +369,7 @@ export class Session {
private readonly voiceLlmModel: string | null;
private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null;
private readonly localSpeechModelsDir: string;
private readonly defaultLocalSpeechModelIds: SherpaOnnxModelId[];
private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[];
private readonly registerVoiceSpeakHandler?: (
agentId: string,
handler: VoiceSpeakHandler
@@ -1894,8 +1892,8 @@ export class Session {
const modelsDir = this.localSpeechModelsDir;
const models = await Promise.all(
listSherpaOnnxModels().map(async (model) => {
const modelDir = getSherpaOnnxModelDir(modelsDir, model.id);
listLocalSpeechModels().map(async (model) => {
const modelDir = getLocalSpeechModelDir(modelsDir, model.id);
const missingFiles: string[] = [];
for (const rel of model.requiredFiles) {
const filePath = join(modelDir, rel);
@@ -1943,8 +1941,8 @@ export class Session {
? msg.modelIds
: this.defaultLocalSpeechModelIds;
const allModelIds = new Set(listSherpaOnnxModels().map((m) => m.id));
const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as SherpaOnnxModelId));
const allModelIds = new Set(listLocalSpeechModels().map((m) => m.id));
const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as LocalSpeechModelId));
if (invalid.length > 0) {
this.emit({
type: "speech_models_download_response",
@@ -1958,9 +1956,9 @@ export class Session {
return;
}
const modelIds = modelIdsRaw as SherpaOnnxModelId[];
const modelIds = modelIdsRaw as LocalSpeechModelId[];
try {
await ensureSherpaOnnxModels({
await ensureLocalSpeechModels({
modelsDir,
modelIds,
autoDownload: true,
packages/server/src/server/speech/providers/local/config.ts
+21 -15
View File
@@ -12,20 +12,24 @@ import {
type LocalSpeechModelId,
type LocalSttModelId,
type LocalTtsModelId,
} from "./sherpa/model-catalog.js";
} from "./models.js";
export type LocalSpeechModelConfig = {
dictationStt: LocalSttModelId;
voiceStt: LocalSttModelId;
voiceTts: LocalTtsModelId;
voiceTtsSpeakerId?: number;
voiceTtsSpeed?: number;
};
export type LocalSpeechProviderConfig = {
modelsDir: string;
autoDownload?: boolean;
models: LocalSpeechModelConfig;
};
export type ResolvedLocalSpeechConfig = {
local: LocalSpeechProviderConfig | undefined;
dictationLocalSttModel: LocalSttModelId;
voiceLocalSttModel: LocalSttModelId;
voiceLocalTtsModel: LocalTtsModelId;
voiceLocalTtsSpeakerId?: number;
voiceLocalTtsSpeed?: number;
};
export type { LocalSpeechModelId, LocalSttModelId, LocalTtsModelId };
@@ -146,16 +150,18 @@ export function resolveLocalSpeechConfig(params: {
? {
modelsDir: parsed.modelsDir,
autoDownload: parsed.autoDownload,
models: {
dictationStt: parsed.dictationLocalSttModel,
voiceStt: parsed.voiceLocalSttModel,
voiceTts: parsed.voiceLocalTtsModel,
...(parsed.voiceLocalTtsSpeakerId !== undefined
? { voiceTtsSpeakerId: parsed.voiceLocalTtsSpeakerId }
: {}),
...(parsed.voiceLocalTtsSpeed !== undefined
? { voiceTtsSpeed: parsed.voiceLocalTtsSpeed }
: {}),
},
}
: undefined,
dictationLocalSttModel: parsed.dictationLocalSttModel,
voiceLocalSttModel: parsed.voiceLocalSttModel,
voiceLocalTtsModel: parsed.voiceLocalTtsModel,
...(parsed.voiceLocalTtsSpeakerId !== undefined
? { voiceLocalTtsSpeakerId: parsed.voiceLocalTtsSpeakerId }
: {}),
...(parsed.voiceLocalTtsSpeed !== undefined
? { voiceLocalTtsSpeed: parsed.voiceLocalTtsSpeed }
: {}),
};
}
packages/server/src/server/speech/providers/local/models.ts
+48
View File
@@ -0,0 +1,48 @@
import {
ensureSherpaOnnxModels,
getSherpaOnnxModelDir,
} from "./sherpa/model-downloader.js";
import {
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
LocalSttModelIdSchema,
LocalTtsModelIdSchema,
listSherpaOnnxModels,
type LocalSpeechModelId,
type LocalSttModelId,
type LocalTtsModelId,
} from "./sherpa/model-catalog.js";
export {
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
LocalSttModelIdSchema,
LocalTtsModelIdSchema,
type LocalSpeechModelId,
type LocalSttModelId,
type LocalTtsModelId,
};
export type LocalSpeechModelSpec = ReturnType<typeof listSherpaOnnxModels>[number];
export function listLocalSpeechModels(): LocalSpeechModelSpec[] {
return listSherpaOnnxModels();
}
export function getLocalSpeechModelDir(modelsDir: string, modelId: LocalSpeechModelId): string {
return getSherpaOnnxModelDir(modelsDir, modelId);
}
export async function ensureLocalSpeechModels(options: {
modelsDir: string;
modelIds: LocalSpeechModelId[];
autoDownload?: boolean;
logger: import("pino").Logger;
}): Promise<Record<LocalSpeechModelId, string>> {
return ensureSherpaOnnxModels({
modelsDir: options.modelsDir,
modelIds: options.modelIds,
autoDownload: options.autoDownload ?? true,
logger: options.logger,
});
}
packages/server/src/server/speech/providers/local/runtime.ts
+14 -16
View File
@@ -5,10 +5,8 @@ import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-pr
import type { RequestedSpeechProviders } from "../../speech-types.js";
import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js";
import {
ensureSherpaOnnxModels,
getSherpaOnnxModelDir,
} from "./sherpa/model-downloader.js";
import {
ensureLocalSpeechModels,
getLocalSpeechModelDir,
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
LocalSttModelIdSchema,
@@ -16,7 +14,7 @@ import {
type LocalSpeechModelId,
type LocalSttModelId,
type LocalTtsModelId,
} from "./sherpa/model-catalog.js";
} from "./models.js";
import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js";
import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
@@ -63,13 +61,13 @@ function resolveConfiguredLocalModels(
): ResolvedLocalModels {
return {
dictationLocalSttModel: LocalSttModelIdSchema.parse(
speechConfig?.localModels.dictationStt ?? DEFAULT_LOCAL_STT_MODEL
speechConfig?.local?.models.dictationStt ?? DEFAULT_LOCAL_STT_MODEL
),
voiceLocalSttModel: LocalSttModelIdSchema.parse(
speechConfig?.localModels.voiceStt ?? DEFAULT_LOCAL_STT_MODEL
speechConfig?.local?.models.voiceStt ?? DEFAULT_LOCAL_STT_MODEL
),
voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
speechConfig?.localModels.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL
speechConfig?.local?.models.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL
),
};
}
@@ -110,7 +108,7 @@ async function createLocalSttEngine(params: {
const { modelId, modelsDir, logger } = params;
if (modelId === "parakeet-tdt-0.6b-v3-int8") {
const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
return {
kind: "offline",
engine: new SherpaOfflineRecognizerEngine(
@@ -131,7 +129,7 @@ async function createLocalSttEngine(params: {
}
if (modelId === "paraformer-bilingual-zh-en") {
const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
return {
kind: "online",
engine: new SherpaOnlineRecognizerEngine(
@@ -151,7 +149,7 @@ async function createLocalSttEngine(params: {
}
if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
return {
kind: "online",
engine: new SherpaOnlineRecognizerEngine(
@@ -204,7 +202,7 @@ export async function initializeLocalSpeechServices(params: {
},
"Ensuring local speech models"
);
await ensureSherpaOnnxModels({
await ensureLocalSpeechModels({
modelsDir: localConfig.modelsDir,
modelIds: requiredLocalModelIds,
autoDownload: localConfig.autoDownload ?? true,
@@ -308,7 +306,7 @@ export async function initializeLocalSpeechServices(params: {
} else {
try {
if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
localVoiceTtsProvider = await PocketTtsOnnxTTS.create(
{
modelDir,
@@ -318,13 +316,13 @@ export async function initializeLocalSpeechServices(params: {
logger
);
} else {
const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
localVoiceTtsProvider = new SherpaOnnxTTS(
{
preset: localModels.voiceLocalTtsModel,
modelDir,
speakerId: speechConfig?.localModels.voiceTtsSpeakerId,
speed: speechConfig?.localModels.voiceTtsSpeed,
speakerId: speechConfig?.local?.models.voiceTtsSpeakerId,
speed: speechConfig?.local?.models.voiceTtsSpeed,
},
logger
);
packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts
+1 -1
View File
@@ -209,7 +209,7 @@ export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] {
export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec {
const spec = SHERPA_ONNX_MODEL_CATALOG[id];
if (!spec) {
throw new Error(`Unknown sherpa-onnx model id: ${id}`);
throw new Error(`Unknown local speech model id: ${id}`);
}
return {
id,
packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts
+2 -2
View File
@@ -96,7 +96,7 @@ async function isNonEmptyFile(filePath: string): Promise<boolean> {
export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOptions): Promise<string> {
const logger = options.logger.child({
module: "speech",
provider: "sherpa-onnx",
provider: "local",
component: "model-downloader",
modelId: options.modelId,
});
@@ -109,7 +109,7 @@ export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOption
if (!options.autoDownload) {
throw new Error(
`Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` +
`Missing local speech model files for ${options.modelId} in ${modelDir}. ` +
`Set PASEO_LOCAL_AUTO_DOWNLOAD=1 to auto-download.`
);
}
packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts
+1 -1
View File
@@ -34,7 +34,7 @@ export class SherpaOfflineRecognizerEngine {
private readonly logger: pino.Logger;
constructor(config: SherpaOfflineRecognizerConfig, logger: pino.Logger) {
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "offline-recognizer" });
this.logger = logger.child({ module: "speech", provider: "local", component: "offline-recognizer" });
assertFileExists(config.model.encoder, "offline encoder");
assertFileExists(config.model.decoder, "offline decoder");
packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts
+1 -1
View File
@@ -46,7 +46,7 @@ export class SherpaOnlineRecognizerEngine {
private readonly logger: pino.Logger;
constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) {
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "online-recognizer" });
this.logger = logger.child({ module: "speech", provider: "local", component: "online-recognizer" });
const { model } = config;
if (model.kind === "transducer") {
packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts
+1 -1
View File
@@ -25,7 +25,7 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider {
constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) {
this.engine = config.engine;
this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" });
this.logger = logger.child({ module: "speech", provider: "local", component: "parakeet-stt" });
}
public createSession(params: {
packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts
+1 -1
View File
@@ -28,7 +28,7 @@ export class SherpaOnnxSTT implements SpeechToTextProvider {
this.engine = config.engine;
this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
this.tailPaddingMs = config.tailPaddingMs ?? 500;
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" });
this.logger = logger.child({ module: "speech", provider: "local", component: "stt" });
}
public createSession(params: {
packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts
+1 -1
View File
@@ -33,7 +33,7 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
}
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "tts" });
this.logger = logger.child({ module: "speech", provider: "local", component: "tts" });
this.speakerId = config.speakerId ?? 0;
this.speed = config.speed ?? 1.0;
packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts
+12 -12
View File
@@ -123,18 +123,18 @@ describe("speech models (download E2E)", () => {
local: {
modelsDir,
autoDownload: false,
},
localModels: {
dictationStt:
set === "parakeet-pocket"
? "parakeet-tdt-0.6b-v3-int8"
: "zipformer-bilingual-zh-en-2023-02-20",
voiceStt:
set === "parakeet-pocket"
? "parakeet-tdt-0.6b-v3-int8"
: "zipformer-bilingual-zh-en-2023-02-20",
voiceTts:
set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
models: {
dictationStt:
set === "parakeet-pocket"
? "parakeet-tdt-0.6b-v3-int8"
: "zipformer-bilingual-zh-en-2023-02-20",
voiceStt:
set === "parakeet-pocket"
? "parakeet-tdt-0.6b-v3-int8"
: "zipformer-bilingual-zh-en-2023-02-20",
voiceTts:
set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
},
},
},
});
packages/server/src/server/speech/speech-config-resolver.test.ts
+23 -11
View File
@@ -33,10 +33,15 @@ describe("resolveSpeechConfig", () => {
expect(result.speech.local).toEqual({
modelsDir: path.join(paseoHome, "models", "local-speech"),
autoDownload: true,
models: {
dictationStt: "parakeet-tdt-0.6b-v3-int8",
voiceStt: "parakeet-tdt-0.6b-v3-int8",
voiceTts: "pocket-tts-onnx-int8",
},
});
expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8");
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
});
test("resolves feature-scoped local model env vars", () => {
@@ -73,6 +78,13 @@ describe("resolveSpeechConfig", () => {
expect(result.speech.local).toEqual({
modelsDir: "/tmp/models",
autoDownload: false,
models: {
dictationStt: "zipformer-bilingual-zh-en-2023-02-20",
voiceStt: "parakeet-tdt-0.6b-v3-int8",
voiceTts: "kitten-nano-en-v0_1-fp16",
voiceTtsSpeakerId: 5,
voiceTtsSpeed: 1.35,
},
});
expect(result.speech.providers.dictationStt).toEqual({
provider: "local",
@@ -86,11 +98,11 @@ describe("resolveSpeechConfig", () => {
provider: "local",
explicit: true,
});
expect(result.speech.localModels.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20");
expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.localModels.voiceTts).toBe("kitten-nano-en-v0_1-fp16");
expect(result.speech.localModels.voiceTtsSpeakerId).toBe(5);
expect(result.speech.localModels.voiceTtsSpeed).toBe(1.35);
expect(result.speech.local?.models.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20");
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16");
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5);
expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35);
expect(result.openai?.apiKey).toBe("env-key");
expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe");
});
@@ -108,8 +120,8 @@ describe("resolveSpeechConfig", () => {
persisted,
});
expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8");
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
});
});
packages/server/src/server/speech/speech-config-resolver.ts
-11
View File
@@ -100,17 +100,6 @@ export function resolveSpeechConfig(params: {
...(local.local
? { local: local.local }
: {}),
localModels: {
dictationStt: local.dictationLocalSttModel,
voiceStt: local.voiceLocalSttModel,
voiceTts: local.voiceLocalTtsModel,
...(local.voiceLocalTtsSpeakerId !== undefined
? { voiceTtsSpeakerId: local.voiceLocalTtsSpeakerId }
: {}),
...(local.voiceLocalTtsSpeed !== undefined
? { voiceTtsSpeed: local.voiceLocalTtsSpeed }
: {}),
},
},
};
}
packages/server/src/server/voice-local-agent.e2e.test.ts
-5
View File
@@ -53,11 +53,6 @@ function waitForSignal<T>(
voiceStt: { provider: "openai", explicit: true },
voiceTts: { provider: "openai", explicit: true },
},
localModels: {
dictationStt: "parakeet-tdt-0.6b-v3-int8",
voiceStt: "parakeet-tdt-0.6b-v3-int8",
voiceTts: "pocket-tts-onnx-int8",
},
},
voiceLlmProvider: "codex",
voiceLlmProviderExplicit: true,
packages/server/src/server/websocket-server.ts
+3 -3
View File
@@ -20,7 +20,7 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js";
import { PushTokenStore } from "./push/token-store.js";
import { PushService } from "./push/push-service.js";
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js";
import type { LocalSpeechModelId } from "./speech/providers/local/models.js";
import type {
VoiceCallerContext,
VoiceMcpStdioConfig,
@@ -79,7 +79,7 @@ export class VoiceAssistantWebSocketServer {
stt?: SpeechToTextProvider | null;
localModels?: {
modelsDir: string;
defaultModelIds: SherpaOnnxModelId[];
defaultModelIds: LocalSpeechModelId[];
};
} | null;
private readonly voice: {
@@ -119,7 +119,7 @@ export class VoiceAssistantWebSocketServer {
stt?: SpeechToTextProvider | null;
localModels?: {
modelsDir: string;
defaultModelIds: SherpaOnnxModelId[];
defaultModelIds: LocalSpeechModelId[];
};
}
) {
packages/website/src/routes/index.tsx
+2 -2
View File
@@ -287,8 +287,8 @@ function FAQ() {
<FAQItem question="Is this free?">
Paseo is free and open source. It wraps CLI tools like Claude Code and
Codex, which you'll need to have installed and configured with your
own credentials. Voice features currently require an OpenAI API key,
but local voice is coming soon.
own credentials. Voice is local-first by default and can optionally use
OpenAI speech providers if you configure them.
</FAQItem>
<FAQItem question="Does my code leave my machine?">
Paseo itself doesn't send your code anywhere. Agents run locally and