Simplify local speech models
This commit is contained in:
Mohamed Boudra
@@ -147,8 +147,8 @@ describe("paseo daemon bootstrap", () => {
|
||||
local: {
|
||||
modelsDir: path.join(os.tmpdir(), `paseo-missing-models-${Date.now()}`),
|
||||
models: {
|
||||
dictationStt: "parakeet-tdt-0.6b-v3-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v3-int8",
|
||||
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceTts: "kokoro-en-v0_19",
|
||||
},
|
||||
},
|
||||
|
||||
@@ -39,35 +39,25 @@ async function readFixture(fileName: string): Promise<Buffer> {
|
||||
return readFile(fixturePath(fileName));
|
||||
}
|
||||
|
||||
function hasSherpaZipformerModels(modelsDir: string): boolean {
|
||||
function hasSherpaParakeetModels(modelsDir: string): boolean {
|
||||
return (
|
||||
existsSync(
|
||||
path.join(
|
||||
modelsDir,
|
||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
"encoder-epoch-99-avg-1.onnx",
|
||||
),
|
||||
path.join(modelsDir, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", "encoder.int8.onnx"),
|
||||
) &&
|
||||
existsSync(
|
||||
path.join(
|
||||
modelsDir,
|
||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
"tokens.txt",
|
||||
),
|
||||
)
|
||||
existsSync(path.join(modelsDir, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", "tokens.txt"))
|
||||
);
|
||||
}
|
||||
|
||||
function hasSherpaKittenModels(modelsDir: string): boolean {
|
||||
function hasSherpaKokoroModels(modelsDir: string): boolean {
|
||||
return (
|
||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "model.fp16.onnx")) &&
|
||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "voices.bin")) &&
|
||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "tokens.txt"))
|
||||
existsSync(path.join(modelsDir, "kokoro-en-v0_19", "model.onnx")) &&
|
||||
existsSync(path.join(modelsDir, "kokoro-en-v0_19", "voices.bin")) &&
|
||||
existsSync(path.join(modelsDir, "kokoro-en-v0_19", "tokens.txt"))
|
||||
);
|
||||
}
|
||||
|
||||
const hasLocalSpeech =
|
||||
hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir);
|
||||
hasSherpaParakeetModels(localModelsDir) && hasSherpaKokoroModels(localModelsDir);
|
||||
const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey);
|
||||
const speechTest = hasAnySpeech ? test : test.skip;
|
||||
|
||||
@@ -492,11 +482,10 @@ function resolveSpeechConfig() {
|
||||
local: {
|
||||
modelsDir: localModelsDir,
|
||||
models: {
|
||||
dictationStt:
|
||||
process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceStt:
|
||||
process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceTts: process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
|
||||
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceTts: "kokoro-en-v0_19",
|
||||
voiceTtsSpeakerId: 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
// Type augmentation for onnxruntime-node to include runtime properties
|
||||
// not exposed in the official type definitions
|
||||
|
||||
import type * as ort from "onnxruntime-node";
|
||||
|
||||
declare module "onnxruntime-node" {
|
||||
interface InferenceSession {
|
||||
/** Input tensor names (available at runtime but not in types) */
|
||||
readonly inputNames: string[];
|
||||
/** Output tensor names (available at runtime but not in types) */
|
||||
readonly outputNames?: string[];
|
||||
/** Input metadata for shape/type info (available at runtime but not in types) */
|
||||
readonly inputMetadata?: unknown;
|
||||
}
|
||||
}
|
||||
@@ -1,629 +0,0 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { Readable } from "node:stream";
|
||||
import type pino from "pino";
|
||||
|
||||
import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js";
|
||||
import {
|
||||
chunkBuffer,
|
||||
float32ToPcm16le,
|
||||
parsePcm16MonoWav,
|
||||
pcm16leToFloat32,
|
||||
} from "../../../audio.js";
|
||||
import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
|
||||
|
||||
type OrtModule = typeof import("onnxruntime-node");
|
||||
type OrtSession = import("onnxruntime-node").InferenceSession;
|
||||
type OrtTensor = import("onnxruntime-node").Tensor;
|
||||
|
||||
function createSessionFeed(feed: Record<string, OrtTensor>): Record<string, OrtTensor> {
|
||||
return feed;
|
||||
}
|
||||
|
||||
interface SentencePieceProcessor {
|
||||
encodeIds: (text: string) => number[];
|
||||
load?: (modelPath: string) => unknown;
|
||||
Load?: (modelPath: string) => unknown;
|
||||
}
|
||||
|
||||
function assertFileExists(filePath: string, label: string): void {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function product(dims: number[]): number {
|
||||
let out = 1;
|
||||
for (const d of dims) out *= d;
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeDims(dims: Array<number | string | null | undefined>): number[] {
|
||||
// ONNX metadata can contain dynamic dimensions as strings (e.g. "batch") or -1.
|
||||
// For state tensors we want a valid minimal shape, so coerce unknown/invalid dims to 1.
|
||||
// Preserve explicit 0 dims (some models use empty initial state buffers with shape [0]).
|
||||
return dims.map((d) => {
|
||||
if (typeof d === "number" && Number.isFinite(d)) {
|
||||
if (d === 0) return 0;
|
||||
if (d > 0) return d;
|
||||
return 1;
|
||||
}
|
||||
return 1;
|
||||
});
|
||||
}
|
||||
|
||||
function getSessionInputMeta(
|
||||
session: OrtSession,
|
||||
inputName: string,
|
||||
): { type?: string; dims?: Array<number | string | null> } | undefined {
|
||||
const metaAny = session.inputMetadata;
|
||||
if (!Array.isArray(metaAny)) {
|
||||
return undefined;
|
||||
}
|
||||
const entry = metaAny.find((m) => m.name === inputName);
|
||||
if (!entry) return undefined;
|
||||
return { type: String(entry.type), dims: entry.shape };
|
||||
}
|
||||
|
||||
function toBigInt64(values: number[]): BigInt64Array {
|
||||
const out = new BigInt64Array(values.length);
|
||||
for (let i = 0; i < values.length; i += 1) {
|
||||
out[i] = BigInt(values[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function randn(): number {
|
||||
// Box–Muller
|
||||
let u = 0;
|
||||
let v = 0;
|
||||
while (u === 0) u = Math.random();
|
||||
while (v === 0) v = Math.random();
|
||||
return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
|
||||
}
|
||||
|
||||
function normalizeTextForPocket(text: string): string {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error("Cannot synthesize empty text");
|
||||
}
|
||||
let out = trimmed;
|
||||
if (out.length > 0 && /[A-Za-z0-9]$/.test(out)) {
|
||||
out = `${out}.`;
|
||||
}
|
||||
if (out.length > 0 && /[a-z]/.test(out[0])) {
|
||||
out = out[0].toUpperCase() + out.slice(1);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function loadOrt(): Promise<OrtModule> {
|
||||
return import("onnxruntime-node");
|
||||
}
|
||||
|
||||
interface SentencePieceModule {
|
||||
SentencePieceProcessor?: new () => SentencePieceProcessor;
|
||||
default?:
|
||||
| (new () => SentencePieceProcessor)
|
||||
| { SentencePieceProcessor?: new () => SentencePieceProcessor };
|
||||
}
|
||||
|
||||
function isSentencePieceModule(mod: unknown): mod is SentencePieceModule {
|
||||
return mod !== null && typeof mod === "object";
|
||||
}
|
||||
|
||||
function getSentencePieceProcessor(
|
||||
mod: SentencePieceModule,
|
||||
): (new () => SentencePieceProcessor) | undefined {
|
||||
if (mod.SentencePieceProcessor) {
|
||||
return mod.SentencePieceProcessor;
|
||||
}
|
||||
const defaultValue = mod.default;
|
||||
if (
|
||||
defaultValue &&
|
||||
typeof defaultValue === "object" &&
|
||||
"SentencePieceProcessor" in defaultValue
|
||||
) {
|
||||
return defaultValue.SentencePieceProcessor;
|
||||
}
|
||||
if (typeof defaultValue === "function") {
|
||||
return defaultValue as new () => SentencePieceProcessor;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function loadSentencePiece(tokenizerModelPath: string): Promise<SentencePieceProcessor> {
|
||||
const mod = await import("@sctg/sentencepiece-js");
|
||||
|
||||
if (!isSentencePieceModule(mod)) {
|
||||
throw new Error("@sctg/sentencepiece-js module has unexpected shape");
|
||||
}
|
||||
|
||||
const Processor = getSentencePieceProcessor(mod);
|
||||
|
||||
if (!Processor) {
|
||||
throw new Error("Failed to load SentencePiece processor from @sctg/sentencepiece-js");
|
||||
}
|
||||
|
||||
const sp = new Processor();
|
||||
|
||||
if (typeof sp.load === "function") {
|
||||
await sp.load(tokenizerModelPath);
|
||||
} else if (typeof sp.Load === "function") {
|
||||
sp.Load(tokenizerModelPath);
|
||||
} else {
|
||||
throw new Error("SentencePiece processor does not expose load()/Load()");
|
||||
}
|
||||
|
||||
return sp;
|
||||
}
|
||||
|
||||
function getOrtProviders(ort: OrtModule, device: "auto" | "cpu" | "cuda"): string[] {
|
||||
// NOTE: onnxruntime-node uses backend names like "cpu"/"coreml"/"webgpu" (not "CPUExecutionProvider").
|
||||
if (device === "cpu") return ["cpu"];
|
||||
if (device === "cuda") return ["cuda", "cpu"];
|
||||
// auto
|
||||
// CoreML EP does not support some dynamic/zero-length shapes used by Pocket TTS (e.g. [1, 0, 32]).
|
||||
// Default to CPU to keep behavior predictable across platforms.
|
||||
void ort;
|
||||
return ["cpu"];
|
||||
}
|
||||
|
||||
function createZeroTensorForInput(
|
||||
ort: OrtModule,
|
||||
session: OrtSession,
|
||||
inputName: string,
|
||||
): OrtTensor {
|
||||
const meta = getSessionInputMeta(session, inputName);
|
||||
const dims = normalizeDims(meta?.dims ?? []);
|
||||
if (dims.length === 0) {
|
||||
throw new Error(`Missing input metadata shape for ${inputName}`);
|
||||
}
|
||||
|
||||
const type = (meta?.type ?? "float32").toLowerCase();
|
||||
const size = product(dims);
|
||||
|
||||
if (type.includes("int64")) {
|
||||
return new ort.Tensor("int64", new BigInt64Array(size), dims);
|
||||
}
|
||||
if (type.includes("bool")) {
|
||||
return new ort.Tensor("bool", new Uint8Array(size), dims);
|
||||
}
|
||||
return new ort.Tensor("float32", new Float32Array(size), dims);
|
||||
}
|
||||
|
||||
function initState(session: OrtSession, ort: OrtModule): Record<string, OrtTensor> {
|
||||
const out: Record<string, OrtTensor> = {};
|
||||
for (const name of session.inputNames) {
|
||||
if (name.startsWith("state_")) {
|
||||
out[name] = createZeroTensorForInput(ort, session, name);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function updateStateFromOutputs(
|
||||
state: Record<string, OrtTensor>,
|
||||
outputs: Record<string, OrtTensor>,
|
||||
): void {
|
||||
for (const [name, tensor] of Object.entries(outputs)) {
|
||||
if (!name.startsWith("out_state_")) continue;
|
||||
const idx = Number.parseInt(name.replace("out_state_", ""), 10);
|
||||
if (Number.isFinite(idx)) {
|
||||
state[`state_${idx}`] = tensor;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface OrtTensorWithData {
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
function tensorDataFloat32(t: OrtTensor): Float32Array {
|
||||
const tensorWithData = t as OrtTensorWithData;
|
||||
const data = tensorWithData.data;
|
||||
if (data instanceof Float32Array) return data;
|
||||
if (Array.isArray(data)) return Float32Array.from(data);
|
||||
throw new Error("Unexpected tensor data type (expected Float32Array)");
|
||||
}
|
||||
|
||||
export interface PocketTtsOnnxConfig {
|
||||
modelDir: string;
|
||||
precision?: "int8" | "fp32";
|
||||
device?: "auto" | "cpu" | "cuda";
|
||||
temperature?: number;
|
||||
lsdSteps?: number;
|
||||
maxFrames?: number;
|
||||
framesAfterEos?: number;
|
||||
firstChunkFrames?: number;
|
||||
maxChunkFrames?: number;
|
||||
targetChunkMs?: number;
|
||||
referenceAudioFile?: string;
|
||||
}
|
||||
|
||||
class PocketTtsOnnxEngine {
|
||||
static readonly SAMPLE_RATE = 24000;
|
||||
static readonly SAMPLES_PER_FRAME = 1920;
|
||||
|
||||
private readonly ort: OrtModule;
|
||||
|
||||
private readonly temperature: number;
|
||||
private readonly lsdSteps: number;
|
||||
private readonly maxFrames: number;
|
||||
private readonly framesAfterEos: number;
|
||||
|
||||
private readonly firstChunkFrames: number;
|
||||
private readonly maxChunkFrames: number;
|
||||
|
||||
private readonly tokenizer: SentencePieceProcessor;
|
||||
private readonly textConditioner: OrtSession;
|
||||
private readonly flowLmMain: OrtSession;
|
||||
private readonly flowLmFlow: OrtSession;
|
||||
private readonly mimiDecoder: OrtSession;
|
||||
|
||||
private readonly stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
|
||||
private readonly voiceEmbeddings: OrtTensor;
|
||||
|
||||
private constructor(args: {
|
||||
ort: OrtModule;
|
||||
temperature: number;
|
||||
lsdSteps: number;
|
||||
maxFrames: number;
|
||||
framesAfterEos: number;
|
||||
firstChunkFrames: number;
|
||||
maxChunkFrames: number;
|
||||
tokenizer: SentencePieceProcessor;
|
||||
textConditioner: OrtSession;
|
||||
flowLmMain: OrtSession;
|
||||
flowLmFlow: OrtSession;
|
||||
mimiDecoder: OrtSession;
|
||||
stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
|
||||
voiceEmbeddings: OrtTensor;
|
||||
}) {
|
||||
this.ort = args.ort;
|
||||
this.temperature = args.temperature;
|
||||
this.lsdSteps = args.lsdSteps;
|
||||
this.maxFrames = args.maxFrames;
|
||||
this.framesAfterEos = args.framesAfterEos;
|
||||
this.firstChunkFrames = args.firstChunkFrames;
|
||||
this.maxChunkFrames = args.maxChunkFrames;
|
||||
this.tokenizer = args.tokenizer;
|
||||
this.textConditioner = args.textConditioner;
|
||||
this.flowLmMain = args.flowLmMain;
|
||||
this.flowLmFlow = args.flowLmFlow;
|
||||
this.mimiDecoder = args.mimiDecoder;
|
||||
this.stBuffers = args.stBuffers;
|
||||
this.voiceEmbeddings = args.voiceEmbeddings;
|
||||
}
|
||||
|
||||
static async create(
|
||||
config: PocketTtsOnnxConfig,
|
||||
logger: pino.Logger,
|
||||
): Promise<PocketTtsOnnxEngine> {
|
||||
const log = logger.child({
|
||||
module: "speech",
|
||||
provider: "pocket-tts",
|
||||
component: "onnx-engine",
|
||||
});
|
||||
|
||||
const modelDir = config.modelDir;
|
||||
const onnxDir = `${modelDir}/onnx`;
|
||||
const precision = config.precision ?? "int8";
|
||||
const device = config.device ?? "auto";
|
||||
const temperature = config.temperature ?? 0.7;
|
||||
const lsdSteps = config.lsdSteps ?? 10;
|
||||
const maxFrames = config.maxFrames ?? 500;
|
||||
const framesAfterEos = config.framesAfterEos ?? 3;
|
||||
const firstChunkFrames = config.firstChunkFrames ?? 2;
|
||||
const maxChunkFrames = config.maxChunkFrames ?? 15;
|
||||
|
||||
const tokenizerPath = `${modelDir}/tokenizer.model`;
|
||||
const referenceAudioFile = config.referenceAudioFile ?? `${modelDir}/reference_sample.wav`;
|
||||
|
||||
const flowMainFile = precision === "int8" ? "flow_lm_main_int8.onnx" : "flow_lm_main.onnx";
|
||||
const flowFlowFile = precision === "int8" ? "flow_lm_flow_int8.onnx" : "flow_lm_flow.onnx";
|
||||
const decoderFile = precision === "int8" ? "mimi_decoder_int8.onnx" : "mimi_decoder.onnx";
|
||||
|
||||
assertFileExists(`${onnxDir}/mimi_encoder.onnx`, "PocketTTS mimi_encoder");
|
||||
assertFileExists(`${onnxDir}/text_conditioner.onnx`, "PocketTTS text_conditioner");
|
||||
assertFileExists(`${onnxDir}/${flowMainFile}`, "PocketTTS flow_lm_main");
|
||||
assertFileExists(`${onnxDir}/${flowFlowFile}`, "PocketTTS flow_lm_flow");
|
||||
assertFileExists(`${onnxDir}/${decoderFile}`, "PocketTTS mimi_decoder");
|
||||
assertFileExists(tokenizerPath, "PocketTTS tokenizer.model");
|
||||
assertFileExists(referenceAudioFile, "PocketTTS reference_sample.wav");
|
||||
|
||||
const ort = await loadOrt();
|
||||
const providers = getOrtProviders(ort, device);
|
||||
|
||||
const [tokenizer, mimiEncoder, textConditioner, flowLmMain, flowLmFlow, mimiDecoder] =
|
||||
await Promise.all([
|
||||
loadSentencePiece(tokenizerPath),
|
||||
ort.InferenceSession.create(`${onnxDir}/mimi_encoder.onnx`, {
|
||||
executionProviders: providers,
|
||||
}),
|
||||
ort.InferenceSession.create(`${onnxDir}/text_conditioner.onnx`, {
|
||||
executionProviders: providers,
|
||||
}),
|
||||
ort.InferenceSession.create(`${onnxDir}/${flowMainFile}`, {
|
||||
executionProviders: providers,
|
||||
}),
|
||||
ort.InferenceSession.create(`${onnxDir}/${flowFlowFile}`, {
|
||||
executionProviders: providers,
|
||||
}),
|
||||
ort.InferenceSession.create(`${onnxDir}/${decoderFile}`, { executionProviders: providers }),
|
||||
]);
|
||||
|
||||
// Precompute flow matching time-step buffers.
|
||||
const stBuffers: Array<{ s: OrtTensor; t: OrtTensor }> = [];
|
||||
for (let j = 0; j < lsdSteps; j += 1) {
|
||||
const s = j / lsdSteps;
|
||||
const t = s + 1.0 / lsdSteps;
|
||||
stBuffers.push({
|
||||
s: new ort.Tensor("float32", new Float32Array([s]), [1, 1]),
|
||||
t: new ort.Tensor("float32", new Float32Array([t]), [1, 1]),
|
||||
});
|
||||
}
|
||||
|
||||
// Precompute reference voice embeddings once.
|
||||
const refWav = await readFile(referenceAudioFile);
|
||||
const parsed = parsePcm16MonoWav(refWav);
|
||||
let pcm16 = parsed.pcm16;
|
||||
if (parsed.sampleRate !== PocketTtsOnnxEngine.SAMPLE_RATE) {
|
||||
const resampler = new Pcm16MonoResampler({
|
||||
inputRate: parsed.sampleRate,
|
||||
outputRate: PocketTtsOnnxEngine.SAMPLE_RATE,
|
||||
});
|
||||
pcm16 = resampler.processChunk(pcm16);
|
||||
}
|
||||
const floatAudio = pcm16leToFloat32(pcm16);
|
||||
const audioTensor = new ort.Tensor("float32", floatAudio, [1, 1, floatAudio.length]);
|
||||
|
||||
const encoded = await mimiEncoder.run({ audio: audioTensor });
|
||||
const firstOutName = mimiEncoder.outputNames?.[0];
|
||||
const encodedRecord = encoded as Record<string, OrtTensor>;
|
||||
const voiceEmb = firstOutName ? encodedRecord[firstOutName] : Object.values(encodedRecord)[0];
|
||||
if (!voiceEmb) {
|
||||
throw new Error("PocketTTS mimi_encoder: missing output");
|
||||
}
|
||||
|
||||
log.info({ precision, device, providers, lsdSteps, temperature }, "PocketTTS ONNX initialized");
|
||||
|
||||
return new PocketTtsOnnxEngine({
|
||||
ort,
|
||||
temperature,
|
||||
lsdSteps,
|
||||
maxFrames,
|
||||
framesAfterEos,
|
||||
firstChunkFrames,
|
||||
maxChunkFrames,
|
||||
tokenizer,
|
||||
textConditioner,
|
||||
flowLmMain,
|
||||
flowLmFlow,
|
||||
mimiDecoder,
|
||||
stBuffers,
|
||||
voiceEmbeddings: voiceEmb,
|
||||
});
|
||||
}
|
||||
|
||||
private tokenize(text: string): OrtTensor {
|
||||
const normalized = normalizeTextForPocket(text);
|
||||
const ids = this.tokenizer.encodeIds(normalized);
|
||||
const data = toBigInt64(ids ?? []);
|
||||
return new this.ort.Tensor("int64", data, [1, data.length]);
|
||||
}
|
||||
|
||||
private async runTextConditioner(tokenIds: OrtTensor): Promise<OrtTensor> {
|
||||
const feed: Record<string, OrtTensor> = { token_ids: tokenIds };
|
||||
const out = await this.textConditioner.run(feed);
|
||||
const firstOutName = this.textConditioner.outputNames?.[0];
|
||||
const outRecord = out as Record<string, OrtTensor>;
|
||||
const t = firstOutName ? outRecord[firstOutName] : Object.values(outRecord)[0];
|
||||
if (!t) throw new Error("PocketTTS text_conditioner: missing output");
|
||||
return t;
|
||||
}
|
||||
|
||||
private async *runFlowLm(textEmbeddings: OrtTensor): AsyncGenerator<Float32Array> {
|
||||
const ort = this.ort;
|
||||
const state = initState(this.flowLmMain, ort);
|
||||
|
||||
const emptySeq = new ort.Tensor("float32", new Float32Array(0), [1, 0, 32]);
|
||||
const emptyText = new ort.Tensor("float32", new Float32Array(0), [1, 0, 1024]);
|
||||
|
||||
// Voice conditioning pass
|
||||
const resVoice = await this.flowLmMain.run(
|
||||
createSessionFeed({
|
||||
sequence: emptySeq,
|
||||
text_embeddings: this.voiceEmbeddings,
|
||||
...state,
|
||||
}),
|
||||
);
|
||||
updateStateFromOutputs(state, resVoice as Record<string, OrtTensor>);
|
||||
|
||||
// Text conditioning pass
|
||||
const resText = await this.flowLmMain.run(
|
||||
createSessionFeed({
|
||||
sequence: emptySeq,
|
||||
text_embeddings: textEmbeddings,
|
||||
...state,
|
||||
}),
|
||||
);
|
||||
updateStateFromOutputs(state, resText as Record<string, OrtTensor>);
|
||||
|
||||
// Autoregressive generation
|
||||
const curr = new Float32Array(32);
|
||||
curr.fill(Number.NaN);
|
||||
let currTensor = new ort.Tensor("float32", curr, [1, 1, 32]);
|
||||
|
||||
const dt = 1.0 / this.lsdSteps;
|
||||
let eosStep: number | null = null;
|
||||
|
||||
for (let step = 0; step < this.maxFrames; step += 1) {
|
||||
const resStep = await this.flowLmMain.run(
|
||||
createSessionFeed({
|
||||
sequence: currTensor,
|
||||
text_embeddings: emptyText,
|
||||
...state,
|
||||
}),
|
||||
);
|
||||
|
||||
const outputNames = this.flowLmMain.outputNames;
|
||||
const resStepRecord = resStep as Record<string, OrtTensor>;
|
||||
const conditioningName = outputNames?.[0] ?? Object.keys(resStepRecord)[0];
|
||||
const eosName = outputNames?.[1] ?? Object.keys(resStepRecord)[1];
|
||||
|
||||
const conditioning = resStepRecord[conditioningName];
|
||||
const eos = resStepRecord[eosName];
|
||||
if (!conditioning || !eos) {
|
||||
throw new Error("PocketTTS flow_lm_main: missing conditioning/EOS outputs");
|
||||
}
|
||||
updateStateFromOutputs(state, resStepRecord);
|
||||
|
||||
const eosData = tensorDataFloat32(eos);
|
||||
if (eosData[0] > -4.0 && eosStep === null) {
|
||||
eosStep = step;
|
||||
}
|
||||
if (eosStep !== null && step >= eosStep + this.framesAfterEos) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Flow matching with external Euler loop.
|
||||
const std = this.temperature > 0 ? Math.sqrt(this.temperature) : 0;
|
||||
const x = new Float32Array(32);
|
||||
if (std > 0) {
|
||||
for (let i = 0; i < x.length; i += 1) {
|
||||
x[i] = randn() * std;
|
||||
}
|
||||
}
|
||||
|
||||
for (const st of this.stBuffers) {
|
||||
const xTensor = new ort.Tensor("float32", x, [1, 32]);
|
||||
const flowOut = await this.flowLmFlow.run(
|
||||
createSessionFeed({
|
||||
c: conditioning,
|
||||
s: st.s,
|
||||
t: st.t,
|
||||
x: xTensor,
|
||||
}),
|
||||
);
|
||||
const first = this.flowLmFlow.outputNames?.[0];
|
||||
const flowOutRecord = flowOut as Record<string, OrtTensor>;
|
||||
const flowTensor = first ? flowOutRecord[first] : Object.values(flowOutRecord)[0];
|
||||
if (!flowTensor) throw new Error("PocketTTS flow_lm_flow: missing output");
|
||||
const delta = tensorDataFloat32(flowTensor);
|
||||
for (let i = 0; i < x.length; i += 1) {
|
||||
x[i] = x[i] + delta[i] * dt;
|
||||
}
|
||||
}
|
||||
|
||||
yield x;
|
||||
currTensor = new ort.Tensor("float32", x, [1, 1, 32]);
|
||||
}
|
||||
}
|
||||
|
||||
private async decodeLatentsChunk(
|
||||
frames: Float32Array[],
|
||||
state: Record<string, OrtTensor>,
|
||||
): Promise<Float32Array> {
|
||||
const ort = this.ort;
|
||||
const frameCount = frames.length;
|
||||
const flattened = new Float32Array(frameCount * 32);
|
||||
for (let i = 0; i < frameCount; i += 1) {
|
||||
flattened.set(frames[i], i * 32);
|
||||
}
|
||||
const latent = new ort.Tensor("float32", flattened, [1, frameCount, 32]);
|
||||
|
||||
const out = await this.mimiDecoder.run(createSessionFeed({ latent, ...state }));
|
||||
const outRecord = out as Record<string, OrtTensor>;
|
||||
updateStateFromOutputs(state, outRecord);
|
||||
|
||||
const firstOutName = this.mimiDecoder.outputNames?.[0];
|
||||
const audioTensor = firstOutName ? outRecord[firstOutName] : Object.values(outRecord)[0];
|
||||
if (!audioTensor) {
|
||||
throw new Error("PocketTTS mimi_decoder: missing audio output");
|
||||
}
|
||||
return tensorDataFloat32(audioTensor);
|
||||
}
|
||||
|
||||
async *streamAudio(text: string): AsyncGenerator<Float32Array> {
|
||||
const tokenIds = this.tokenize(text);
|
||||
const textEmb = await this.runTextConditioner(tokenIds);
|
||||
|
||||
const decoderState = initState(this.mimiDecoder, this.ort);
|
||||
|
||||
const generated: Float32Array[] = [];
|
||||
let decodedFrames = 0;
|
||||
|
||||
for await (const latent of this.runFlowLm(textEmb)) {
|
||||
generated.push(latent);
|
||||
const pending = generated.length - decodedFrames;
|
||||
|
||||
let chunkSize = 0;
|
||||
if (decodedFrames === 0) {
|
||||
if (pending >= this.firstChunkFrames) {
|
||||
chunkSize = this.firstChunkFrames;
|
||||
}
|
||||
} else if (pending >= this.maxChunkFrames) {
|
||||
chunkSize = this.maxChunkFrames;
|
||||
}
|
||||
|
||||
if (chunkSize > 0) {
|
||||
const audio = await this.decodeLatentsChunk(
|
||||
generated.slice(decodedFrames, decodedFrames + chunkSize),
|
||||
decoderState,
|
||||
);
|
||||
decodedFrames += chunkSize;
|
||||
yield audio;
|
||||
}
|
||||
}
|
||||
|
||||
if (decodedFrames < generated.length) {
|
||||
const audio = await this.decodeLatentsChunk(generated.slice(decodedFrames), decoderState);
|
||||
yield audio;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class PocketTtsOnnxTTS implements TextToSpeechProvider {
|
||||
private readonly engine: PocketTtsOnnxEngine;
|
||||
private readonly chunkMs: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
private constructor(engine: PocketTtsOnnxEngine, logger: pino.Logger, chunkMs: number) {
|
||||
this.engine = engine;
|
||||
this.chunkMs = chunkMs;
|
||||
this.logger = logger.child({ module: "speech", provider: "pocket-tts", component: "tts" });
|
||||
}
|
||||
|
||||
static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise<PocketTtsOnnxTTS> {
|
||||
const engine = await PocketTtsOnnxEngine.create(config, logger);
|
||||
const chunkMs = config.targetChunkMs ?? 50;
|
||||
return new PocketTtsOnnxTTS(engine, logger, chunkMs);
|
||||
}
|
||||
|
||||
async synthesizeSpeech(text: string): Promise<SpeechStreamResult> {
|
||||
const start = Date.now();
|
||||
const sampleRate = PocketTtsOnnxEngine.SAMPLE_RATE;
|
||||
|
||||
const pcmChunkBytes = Math.max(2, Math.round((sampleRate * this.chunkMs) / 1000) * 2);
|
||||
|
||||
const iterable = (async function* (engine: PocketTtsOnnxEngine) {
|
||||
for await (const floatChunk of engine.streamAudio(text)) {
|
||||
const pcm = float32ToPcm16le(floatChunk);
|
||||
for (const chunk of chunkBuffer(pcm, pcmChunkBytes)) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
})(this.engine);
|
||||
|
||||
this.logger.debug(
|
||||
{ ms: Date.now() - start, textLength: text.length },
|
||||
"PocketTTS stream ready",
|
||||
);
|
||||
|
||||
return {
|
||||
stream: Readable.from(iterable),
|
||||
format: `pcm;rate=${sampleRate}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import type { PaseoSpeechConfig } from "../../../bootstrap.js";
|
||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
|
||||
import type { RequestedSpeechProviders } from "../../speech-types.js";
|
||||
import type { TurnDetectionProvider } from "../../turn-detection-provider.js";
|
||||
import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js";
|
||||
import {
|
||||
getLocalSpeechModelDir,
|
||||
DEFAULT_LOCAL_STT_MODEL,
|
||||
@@ -16,20 +15,15 @@ import {
|
||||
type LocalTtsModelId,
|
||||
} from "./models.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js";
|
||||
import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
|
||||
import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
|
||||
import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js";
|
||||
import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js";
|
||||
import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
|
||||
import {
|
||||
ensureSileroVadModel,
|
||||
SherpaSileroTurnDetectionProvider,
|
||||
} from "./sherpa/silero-vad-provider.js";
|
||||
|
||||
type LocalSttEngine =
|
||||
| { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
|
||||
| { kind: "online"; engine: SherpaOnlineRecognizerEngine };
|
||||
type LocalSttEngine = SherpaOfflineRecognizerEngine;
|
||||
|
||||
interface ResolvedLocalModels {
|
||||
dictationLocalSttModel: LocalSttModelId;
|
||||
@@ -117,70 +111,21 @@ async function createLocalSttEngine(params: {
|
||||
}): Promise<LocalSttEngine> {
|
||||
const { modelId, modelsDir, logger } = params;
|
||||
|
||||
if (modelId === "parakeet-tdt-0.6b-v3-int8" || modelId === "parakeet-tdt-0.6b-v2-int8") {
|
||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "offline",
|
||||
engine: new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
joiner: `${modelDir}/joiner.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (modelId === "paraformer-bilingual-zh-en") {
|
||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "online",
|
||||
engine: new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "paraformer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
|
||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "online",
|
||||
engine: new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "transducer",
|
||||
encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
|
||||
decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
|
||||
joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
modelType: "zipformer",
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported local STT model '${modelId}'`);
|
||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
||||
return new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
joiner: `${modelDir}/joiner.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
}
|
||||
|
||||
type LocalConfig = NonNullable<PaseoSpeechConfig["local"]>;
|
||||
@@ -219,13 +164,7 @@ async function initializeLocalVoiceStt(params: {
|
||||
return null;
|
||||
}
|
||||
const voiceEngine = await getLocalSttEngine(modelId);
|
||||
if (voiceEngine?.kind === "offline") {
|
||||
return new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
|
||||
}
|
||||
if (voiceEngine?.kind === "online") {
|
||||
return new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
|
||||
}
|
||||
return null;
|
||||
return voiceEngine ? new SherpaOnnxParakeetSTT({ engine: voiceEngine }, logger) : null;
|
||||
}
|
||||
|
||||
async function initializeLocalDictationStt(params: {
|
||||
@@ -243,18 +182,11 @@ async function initializeLocalDictationStt(params: {
|
||||
return null;
|
||||
}
|
||||
const dictationEngine = await getLocalSttEngine(modelId);
|
||||
if (dictationEngine?.kind === "offline") {
|
||||
if (dictationEngine) {
|
||||
return {
|
||||
id: "local",
|
||||
createSession: () =>
|
||||
new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
||||
};
|
||||
}
|
||||
if (dictationEngine?.kind === "online") {
|
||||
return {
|
||||
id: "local",
|
||||
createSession: () =>
|
||||
new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
||||
new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine }),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
@@ -276,16 +208,6 @@ async function initializeLocalVoiceTts(params: {
|
||||
}
|
||||
try {
|
||||
const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
|
||||
if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
|
||||
return await PocketTtsOnnxTTS.create(
|
||||
{
|
||||
modelDir,
|
||||
precision: "int8",
|
||||
targetChunkMs: 50,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
}
|
||||
return new SherpaOnnxTTS(
|
||||
{
|
||||
preset: localModels.voiceLocalTtsModel,
|
||||
@@ -401,7 +323,7 @@ export async function initializeLocalSpeechServices(params: {
|
||||
maybeFreeable.free();
|
||||
}
|
||||
for (const engine of localSttEngines.values()) {
|
||||
engine.engine.free();
|
||||
engine.free();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,44 +1,19 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts";
|
||||
export type SherpaOnnxModelKind = "stt-offline" | "tts";
|
||||
|
||||
type DefaultModelRole = "stt" | "tts";
|
||||
|
||||
interface SherpaOnnxCatalogEntry {
|
||||
kind: SherpaOnnxModelKind;
|
||||
archiveUrl?: string;
|
||||
downloadFiles?: Array<{ url: string; relPath: string }>;
|
||||
archiveUrl: string;
|
||||
extractedDir: string;
|
||||
requiredFiles: string[];
|
||||
description: string;
|
||||
aliases?: readonly string[];
|
||||
defaultFor?: DefaultModelRole;
|
||||
}
|
||||
|
||||
export const SHERPA_ONNX_MODEL_CATALOG = {
|
||||
"zipformer-bilingual-zh-en-2023-02-20": {
|
||||
kind: "stt-online",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2",
|
||||
extractedDir: "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
requiredFiles: [
|
||||
"encoder-epoch-99-avg-1.onnx",
|
||||
"decoder-epoch-99-avg-1.onnx",
|
||||
"joiner-epoch-99-avg-1.onnx",
|
||||
"tokens.txt",
|
||||
],
|
||||
description: "Streaming Zipformer transducer (fast, good accuracy).",
|
||||
aliases: ["zipformer", "zipformer-bilingual"],
|
||||
},
|
||||
"paraformer-bilingual-zh-en": {
|
||||
kind: "stt-online",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2",
|
||||
extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en",
|
||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"],
|
||||
description: "Streaming Paraformer (often strong accuracy; heavier).",
|
||||
aliases: ["paraformer"],
|
||||
},
|
||||
"parakeet-tdt-0.6b-v2-int8": {
|
||||
kind: "stt-offline",
|
||||
archiveUrl:
|
||||
@@ -46,27 +21,8 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
|
||||
extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
|
||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
|
||||
description: "NVIDIA Parakeet TDT v2 (offline NeMo transducer, English).",
|
||||
aliases: ["parakeet-v2", "parakeet-tdt-v2"],
|
||||
defaultFor: "stt",
|
||||
},
|
||||
"parakeet-tdt-0.6b-v3-int8": {
|
||||
kind: "stt-offline",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
|
||||
extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
|
||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
|
||||
description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).",
|
||||
aliases: ["parakeet", "parakeet-v3", "parakeet-tdt"],
|
||||
},
|
||||
"kitten-nano-en-v0_1-fp16": {
|
||||
kind: "tts",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2",
|
||||
extractedDir: "kitten-nano-en-v0_1-fp16",
|
||||
requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
|
||||
description: "KittenTTS (small, fast English TTS).",
|
||||
aliases: ["kitten"],
|
||||
},
|
||||
"kokoro-en-v0_19": {
|
||||
kind: "tts",
|
||||
archiveUrl:
|
||||
@@ -74,54 +30,8 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
|
||||
extractedDir: "kokoro-en-v0_19",
|
||||
requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
|
||||
description: "Kokoro TTS (higher quality; larger).",
|
||||
aliases: ["kokoro"],
|
||||
defaultFor: "tts",
|
||||
},
|
||||
"pocket-tts-onnx-int8": {
|
||||
kind: "tts",
|
||||
extractedDir: "pocket-tts-onnx-int8",
|
||||
downloadFiles: [
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_encoder.onnx",
|
||||
relPath: "onnx/mimi_encoder.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/text_conditioner.onnx",
|
||||
relPath: "onnx/text_conditioner.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_main_int8.onnx",
|
||||
relPath: "onnx/flow_lm_main_int8.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_flow_int8.onnx",
|
||||
relPath: "onnx/flow_lm_flow_int8.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_decoder_int8.onnx",
|
||||
relPath: "onnx/mimi_decoder_int8.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/tokenizer.model",
|
||||
relPath: "tokenizer.model",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/reference_sample.wav",
|
||||
relPath: "reference_sample.wav",
|
||||
},
|
||||
],
|
||||
requiredFiles: [
|
||||
"onnx/mimi_encoder.onnx",
|
||||
"onnx/text_conditioner.onnx",
|
||||
"onnx/flow_lm_main_int8.onnx",
|
||||
"onnx/flow_lm_flow_int8.onnx",
|
||||
"onnx/mimi_decoder_int8.onnx",
|
||||
"tokenizer.model",
|
||||
"reference_sample.wav",
|
||||
],
|
||||
description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).",
|
||||
aliases: ["pocket", "pocket-tts"],
|
||||
},
|
||||
} as const satisfies Record<string, SherpaOnnxCatalogEntry>;
|
||||
|
||||
export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG;
|
||||
@@ -131,7 +41,7 @@ type ModelIdByKind<K extends SherpaOnnxModelKind> = {
|
||||
[Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K ? Id : never;
|
||||
}[SherpaOnnxModelId];
|
||||
|
||||
export type LocalSttModelId = ModelIdByKind<"stt-online"> | ModelIdByKind<"stt-offline">;
|
||||
export type LocalSttModelId = ModelIdByKind<"stt-offline">;
|
||||
export type LocalTtsModelId = ModelIdByKind<"tts">;
|
||||
|
||||
const ALL_MODEL_IDS: SherpaOnnxModelId[] = Object.keys(SHERPA_ONNX_MODEL_CATALOG).filter(
|
||||
@@ -166,48 +76,22 @@ function resolveDefaultModelId(role: DefaultModelRole): SherpaOnnxModelId {
|
||||
export const DEFAULT_LOCAL_STT_MODEL = resolveDefaultModelId("stt");
|
||||
export const DEFAULT_LOCAL_TTS_MODEL = resolveDefaultModelId("tts");
|
||||
|
||||
function buildAliasMap<T extends SherpaOnnxModelId>(modelIds: readonly T[]): Record<string, T> {
|
||||
const aliasMap: Record<string, T> = {};
|
||||
for (const modelId of modelIds) {
|
||||
const aliases = SHERPA_ONNX_MODEL_CATALOG[modelId].aliases ?? [];
|
||||
for (const alias of aliases) {
|
||||
aliasMap[alias.trim().toLowerCase()] = modelId;
|
||||
}
|
||||
}
|
||||
return aliasMap;
|
||||
}
|
||||
|
||||
function createAliasedModelIdSchema<T extends string>(params: {
|
||||
modelIds: readonly T[];
|
||||
aliases: Record<string, T>;
|
||||
}): z.ZodType<T, z.ZodTypeDef, string> {
|
||||
const validIds = new Set<string>(params.modelIds);
|
||||
function createModelIdSchema<T extends string>(
|
||||
modelIds: readonly T[],
|
||||
): z.ZodType<T, z.ZodTypeDef, string> {
|
||||
const validIds = new Set<string>(modelIds);
|
||||
return z
|
||||
.string()
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.refine(
|
||||
(value): value is T =>
|
||||
validIds.has(value) || Object.prototype.hasOwnProperty.call(params.aliases, value),
|
||||
{
|
||||
message: "Invalid model id",
|
||||
},
|
||||
)
|
||||
.transform((value) => params.aliases[value] ?? value);
|
||||
.refine((value) => validIds.has(value), {
|
||||
message: "Invalid model id",
|
||||
})
|
||||
.transform((value) => value as T);
|
||||
}
|
||||
|
||||
const STT_MODEL_ALIASES = buildAliasMap(LOCAL_STT_MODEL_IDS);
|
||||
const TTS_MODEL_ALIASES = buildAliasMap(LOCAL_TTS_MODEL_IDS);
|
||||
|
||||
export const LocalSttModelIdSchema = createAliasedModelIdSchema({
|
||||
modelIds: LOCAL_STT_MODEL_IDS,
|
||||
aliases: STT_MODEL_ALIASES,
|
||||
});
|
||||
|
||||
export const LocalTtsModelIdSchema = createAliasedModelIdSchema({
|
||||
modelIds: LOCAL_TTS_MODEL_IDS,
|
||||
aliases: TTS_MODEL_ALIASES,
|
||||
});
|
||||
export const LocalSttModelIdSchema = createModelIdSchema(LOCAL_STT_MODEL_IDS);
|
||||
export const LocalTtsModelIdSchema = createModelIdSchema(LOCAL_TTS_MODEL_IDS);
|
||||
|
||||
export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & {
|
||||
id: SherpaOnnxModelId;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -15,69 +15,27 @@ const logger = pino({ level: "silent" });
|
||||
describe("sherpa model downloader", () => {
|
||||
test("getSherpaOnnxModelDir maps modelId to extractedDir", () => {
|
||||
const modelsDir = "/tmp/models";
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8")).toContain(
|
||||
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
|
||||
);
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8")).toContain(
|
||||
"pocket-tts-onnx-int8",
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8")).toContain(
|
||||
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
|
||||
);
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19")).toContain("kokoro-en-v0_19");
|
||||
});
|
||||
|
||||
test("ensureSherpaOnnxModel succeeds without downloading when files exist", async () => {
|
||||
const modelsDir = makeTmpDir();
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "kitten-nano-en-v0_1-fp16");
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19");
|
||||
|
||||
mkdirSync(path.join(modelDir, "espeak-ng-data"), { recursive: true });
|
||||
writeFileSync(path.join(modelDir, "model.fp16.onnx"), "x");
|
||||
writeFileSync(path.join(modelDir, "model.onnx"), "x");
|
||||
writeFileSync(path.join(modelDir, "voices.bin"), "x");
|
||||
writeFileSync(path.join(modelDir, "tokens.txt"), "x");
|
||||
|
||||
const out = await ensureSherpaOnnxModel({
|
||||
modelsDir,
|
||||
modelId: "kitten-nano-en-v0_1-fp16",
|
||||
modelId: "kokoro-en-v0_19",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(out).toBe(modelDir);
|
||||
});
|
||||
|
||||
test("ensureSherpaOnnxModel logs lifecycle events without progress spam", async () => {
|
||||
const modelsDir = makeTmpDir();
|
||||
const infoMessages: string[] = [];
|
||||
|
||||
const loggerWithSpy = {
|
||||
child: () => loggerWithSpy,
|
||||
info: (_obj?: unknown, msg?: string) => {
|
||||
if (typeof msg === "string") {
|
||||
infoMessages.push(msg);
|
||||
}
|
||||
},
|
||||
error: () => undefined,
|
||||
} as unknown as pino.Logger;
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
const payload = Buffer.alloc(128 * 1024, 7);
|
||||
const fetchMock = vi.fn(async () => {
|
||||
return new Response(payload, {
|
||||
status: 200,
|
||||
headers: { "content-length": String(payload.length) },
|
||||
});
|
||||
});
|
||||
globalThis.fetch = fetchMock as typeof fetch;
|
||||
|
||||
try {
|
||||
await ensureSherpaOnnxModel({
|
||||
modelsDir,
|
||||
modelId: "pocket-tts-onnx-int8",
|
||||
logger: loggerWithSpy,
|
||||
});
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
|
||||
expect(fetchMock).toHaveBeenCalled();
|
||||
expect(infoMessages).toContain("Starting model download");
|
||||
expect(infoMessages).toContain("Model download completed");
|
||||
expect(infoMessages).not.toContain("Downloading model artifact");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -111,92 +111,55 @@ export async function ensureSherpaOnnxModel(
|
||||
logger.info({ modelsDir: options.modelsDir }, "Starting model download");
|
||||
|
||||
try {
|
||||
if (spec.archiveUrl) {
|
||||
const downloadsDir = path.join(options.modelsDir, ".downloads");
|
||||
const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
|
||||
const archivePath = path.join(downloadsDir, archiveFilename);
|
||||
const downloadsDir = path.join(options.modelsDir, ".downloads");
|
||||
const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
|
||||
const archivePath = path.join(downloadsDir, archiveFilename);
|
||||
|
||||
if (!(await isNonEmptyFile(archivePath))) {
|
||||
await downloadToFile({
|
||||
url: spec.archiveUrl,
|
||||
outputPath: archivePath,
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
archivePath,
|
||||
modelDir,
|
||||
},
|
||||
"Extracting model archive",
|
||||
);
|
||||
await extractTarArchive(archivePath, options.modelsDir);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
modelDir,
|
||||
},
|
||||
"Verifying downloaded model files",
|
||||
);
|
||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
||||
throw new Error(
|
||||
`Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`,
|
||||
);
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
archivePath,
|
||||
},
|
||||
"Finalizing model artifacts",
|
||||
);
|
||||
try {
|
||||
await rm(archivePath, { force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
logger.info({ modelDir }, "Model download completed");
|
||||
return modelDir;
|
||||
if (!(await isNonEmptyFile(archivePath))) {
|
||||
await downloadToFile({
|
||||
url: spec.archiveUrl,
|
||||
outputPath: archivePath,
|
||||
});
|
||||
}
|
||||
|
||||
if (spec.downloadFiles && spec.downloadFiles.length > 0) {
|
||||
await mkdir(modelDir, { recursive: true });
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
archivePath,
|
||||
modelDir,
|
||||
},
|
||||
"Extracting model archive",
|
||||
);
|
||||
await extractTarArchive(archivePath, options.modelsDir);
|
||||
|
||||
await Promise.all(
|
||||
spec.downloadFiles.map(async (file) => {
|
||||
const dst = path.join(modelDir, file.relPath);
|
||||
if (await isNonEmptyFile(dst)) {
|
||||
return;
|
||||
}
|
||||
await downloadToFile({
|
||||
url: file.url,
|
||||
outputPath: dst,
|
||||
});
|
||||
}),
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
modelDir,
|
||||
},
|
||||
"Verifying downloaded model files",
|
||||
);
|
||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
||||
throw new Error(
|
||||
`Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`,
|
||||
);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
modelDir,
|
||||
},
|
||||
"Verifying downloaded model files",
|
||||
);
|
||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
||||
throw new Error(
|
||||
`Downloaded files for ${options.modelId}, but required files are still missing in ${modelDir}.`,
|
||||
);
|
||||
}
|
||||
|
||||
logger.info({ modelDir }, "Model download completed");
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
throw new Error(`Model spec for ${options.modelId} has no archiveUrl or downloadFiles`);
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
archivePath,
|
||||
},
|
||||
"Finalizing model artifacts",
|
||||
);
|
||||
try {
|
||||
await rm(archivePath, { force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
logger.info({ modelDir }, "Model download completed");
|
||||
return modelDir;
|
||||
} catch (error) {
|
||||
logger.error({ err: error }, "Model download failed");
|
||||
throw error;
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import type pino from "pino";
|
||||
|
||||
import { loadSherpaOnnx } from "./sherpa-onnx-loader.js";
|
||||
|
||||
export type SherpaOnlineRecognizerModel =
|
||||
| {
|
||||
kind: "transducer";
|
||||
encoder: string;
|
||||
decoder: string;
|
||||
joiner: string;
|
||||
tokens: string;
|
||||
modelType?: string;
|
||||
}
|
||||
| {
|
||||
kind: "paraformer";
|
||||
encoder: string;
|
||||
decoder: string;
|
||||
tokens: string;
|
||||
};
|
||||
|
||||
export interface SherpaOnlineRecognizerConfig {
|
||||
model: SherpaOnlineRecognizerModel;
|
||||
numThreads?: number;
|
||||
provider?: "cpu";
|
||||
debug?: 0 | 1;
|
||||
sampleRate?: number;
|
||||
featureDim?: number;
|
||||
decodingMethod?: "greedy_search";
|
||||
maxActivePaths?: number;
|
||||
enableEndpoint?: 0 | 1;
|
||||
rule1MinTrailingSilence?: number;
|
||||
rule2MinTrailingSilence?: number;
|
||||
rule3MinUtteranceLength?: number;
|
||||
}
|
||||
|
||||
function assertFileExists(filePath: string, label: string): void {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateModelFiles(model: SherpaOnlineRecognizerModel): void {
|
||||
if (model.kind === "transducer") {
|
||||
assertFileExists(model.encoder, "transducer encoder");
|
||||
assertFileExists(model.decoder, "transducer decoder");
|
||||
assertFileExists(model.joiner, "transducer joiner");
|
||||
assertFileExists(model.tokens, "tokens");
|
||||
} else {
|
||||
assertFileExists(model.encoder, "paraformer encoder");
|
||||
assertFileExists(model.decoder, "paraformer decoder");
|
||||
assertFileExists(model.tokens, "tokens");
|
||||
}
|
||||
}
|
||||
|
||||
function buildModelConfig(model: SherpaOnlineRecognizerModel): object {
|
||||
if (model.kind === "transducer") {
|
||||
return {
|
||||
transducer: {
|
||||
encoder: model.encoder,
|
||||
decoder: model.decoder,
|
||||
joiner: model.joiner,
|
||||
},
|
||||
tokens: model.tokens,
|
||||
modelType: model.modelType ?? "zipformer",
|
||||
};
|
||||
}
|
||||
return {
|
||||
paraformer: {
|
||||
encoder: model.encoder,
|
||||
decoder: model.decoder,
|
||||
},
|
||||
tokens: model.tokens,
|
||||
};
|
||||
}
|
||||
|
||||
export interface SherpaOnlineStreamNative {
|
||||
acceptWaveform: (sampleRate: number, samples: Float32Array) => void;
|
||||
free?: () => void;
|
||||
}
|
||||
|
||||
export interface SherpaOnlineRecognizerNative {
|
||||
config?: { featConfig?: { sampleRate?: number } };
|
||||
createStream: () => SherpaOnlineStreamNative;
|
||||
isReady: (stream: SherpaOnlineStreamNative) => boolean;
|
||||
decode: (stream: SherpaOnlineStreamNative) => void;
|
||||
getResult: (stream: SherpaOnlineStreamNative) => { text?: string } | string | undefined;
|
||||
reset?: (stream: SherpaOnlineStreamNative) => void;
|
||||
free?: () => void;
|
||||
}
|
||||
|
||||
export class SherpaOnlineRecognizerEngine {
|
||||
public readonly recognizer: SherpaOnlineRecognizerNative;
|
||||
public readonly sampleRate: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) {
|
||||
this.logger = logger.child({
|
||||
module: "speech",
|
||||
provider: "local",
|
||||
component: "online-recognizer",
|
||||
});
|
||||
|
||||
const { model } = config;
|
||||
validateModelFiles(model);
|
||||
|
||||
const sherpa = loadSherpaOnnx();
|
||||
const modelConfig = buildModelConfig(model);
|
||||
|
||||
const featConfig = {
|
||||
sampleRate: config.sampleRate ?? 16000,
|
||||
featureDim: config.featureDim ?? 80,
|
||||
};
|
||||
|
||||
const recognizerConfig = {
|
||||
featConfig,
|
||||
modelConfig: {
|
||||
...modelConfig,
|
||||
// NOTE: In the WASM-backed `sherpa-onnx` npm package, online recognizers
|
||||
// error when `numThreads > 1`. Keep the default conservative.
|
||||
numThreads: config.numThreads ?? 1,
|
||||
provider: config.provider ?? "cpu",
|
||||
debug: config.debug ?? 0,
|
||||
},
|
||||
decodingMethod: config.decodingMethod ?? "greedy_search",
|
||||
maxActivePaths: config.maxActivePaths ?? 4,
|
||||
enableEndpoint: config.enableEndpoint ?? 0,
|
||||
rule1MinTrailingSilence: config.rule1MinTrailingSilence ?? 2.4,
|
||||
rule2MinTrailingSilence: config.rule2MinTrailingSilence ?? 1.2,
|
||||
rule3MinUtteranceLength: config.rule3MinUtteranceLength ?? 20,
|
||||
};
|
||||
|
||||
this.recognizer = sherpa.createOnlineRecognizer(
|
||||
recognizerConfig,
|
||||
) as SherpaOnlineRecognizerNative;
|
||||
const sr = this.recognizer?.config?.featConfig?.sampleRate;
|
||||
this.sampleRate =
|
||||
typeof sr === "number" && Number.isFinite(sr) && sr > 0 ? sr : featConfig.sampleRate;
|
||||
|
||||
this.logger.info(
|
||||
{ sampleRate: this.sampleRate, modelKind: model.kind, numThreads: config.numThreads ?? 2 },
|
||||
"Sherpa online recognizer initialized",
|
||||
);
|
||||
}
|
||||
|
||||
createStream(): SherpaOnlineStreamNative {
|
||||
return this.recognizer.createStream();
|
||||
}
|
||||
|
||||
free(): void {
|
||||
try {
|
||||
this.recognizer?.free?.();
|
||||
} catch (err) {
|
||||
this.logger.warn({ err }, "Failed to free sherpa recognizer");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
export interface SherpaOnnxModule {
|
||||
createOnlineRecognizer: (config: unknown) => unknown;
|
||||
createOfflineRecognizer: (config: unknown) => unknown;
|
||||
createOfflineTts: (config: unknown) => unknown;
|
||||
}
|
||||
|
||||
let cached: SherpaOnnxModule | null = null;
|
||||
|
||||
export function loadSherpaOnnx(): SherpaOnnxModule {
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
cached = require("sherpa-onnx") as SherpaOnnxModule;
|
||||
return cached;
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
import type { StreamingTranscriptionSession } from "../../../speech-provider.js";
|
||||
import { pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js";
|
||||
import {
|
||||
SherpaOnlineRecognizerEngine,
|
||||
type SherpaOnlineStreamNative,
|
||||
} from "./sherpa-online-recognizer.js";
|
||||
|
||||
export class SherpaRealtimeTranscriptionSession
|
||||
extends EventEmitter
|
||||
implements StreamingTranscriptionSession
|
||||
{
|
||||
private readonly engine: SherpaOnlineRecognizerEngine;
|
||||
private stream: SherpaOnlineStreamNative | null = null;
|
||||
private connected = false;
|
||||
|
||||
public readonly requiredSampleRate: number;
|
||||
private currentSegmentId: string | null = null;
|
||||
private previousSegmentId: string | null = null;
|
||||
private lastPartialText = "";
|
||||
private readonly tailPaddingMs: number;
|
||||
|
||||
constructor(params: { engine: SherpaOnlineRecognizerEngine; tailPaddingMs?: number }) {
|
||||
super();
|
||||
this.engine = params.engine;
|
||||
this.requiredSampleRate = this.engine.sampleRate;
|
||||
this.tailPaddingMs = params.tailPaddingMs ?? 500;
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
if (this.connected) {
|
||||
return;
|
||||
}
|
||||
this.stream = this.engine.createStream();
|
||||
this.currentSegmentId = uuidv4();
|
||||
this.connected = true;
|
||||
}
|
||||
|
||||
appendPcm16(pcm16le: Buffer): void {
|
||||
if (!this.connected || !this.stream || !this.currentSegmentId) {
|
||||
this.emit("error", new Error("Sherpa realtime session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const peak = pcm16lePeakAbs(pcm16le);
|
||||
const peakFloat = peak / 32768.0;
|
||||
const targetPeak = 0.6;
|
||||
const maxGain = 50;
|
||||
const gain =
|
||||
peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
|
||||
const floatSamples = pcm16leToFloat32(pcm16le, gain);
|
||||
this.stream.acceptWaveform(this.engine.sampleRate, floatSamples);
|
||||
|
||||
while (this.engine.recognizer.isReady(this.stream)) {
|
||||
this.engine.recognizer.decode(this.stream);
|
||||
}
|
||||
|
||||
const rawResult = this.engine.recognizer.getResult(this.stream);
|
||||
const text = (
|
||||
(typeof rawResult === "object" && rawResult && "text" in rawResult
|
||||
? rawResult.text
|
||||
: undefined) ?? ""
|
||||
).trim();
|
||||
if (text !== this.lastPartialText) {
|
||||
this.lastPartialText = text;
|
||||
this.emit("transcript", {
|
||||
segmentId: this.currentSegmentId,
|
||||
transcript: text,
|
||||
isFinal: false,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
commit(): void {
|
||||
if (!this.connected || !this.stream || !this.currentSegmentId) {
|
||||
this.emit("error", new Error("Sherpa realtime session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const padSamples = Math.max(
|
||||
0,
|
||||
Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000),
|
||||
);
|
||||
if (padSamples > 0) {
|
||||
this.stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
|
||||
}
|
||||
|
||||
while (this.engine.recognizer.isReady(this.stream)) {
|
||||
this.engine.recognizer.decode(this.stream);
|
||||
}
|
||||
|
||||
const rawFinal = this.engine.recognizer.getResult(this.stream);
|
||||
const finalText = (
|
||||
(typeof rawFinal === "object" && rawFinal && "text" in rawFinal
|
||||
? rawFinal.text
|
||||
: undefined) ?? ""
|
||||
).trim();
|
||||
const segmentId = this.currentSegmentId;
|
||||
const previousSegmentId = this.previousSegmentId;
|
||||
|
||||
this.emit("committed", { segmentId, previousSegmentId });
|
||||
this.emit("transcript", { segmentId, transcript: finalText, isFinal: true });
|
||||
|
||||
this.previousSegmentId = segmentId;
|
||||
this.currentSegmentId = uuidv4();
|
||||
this.lastPartialText = "";
|
||||
this.engine.recognizer.reset?.(this.stream);
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
if (!this.connected || !this.stream) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.engine.recognizer.reset?.(this.stream);
|
||||
this.currentSegmentId = uuidv4();
|
||||
this.lastPartialText = "";
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (!this.stream) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.stream.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
} finally {
|
||||
this.stream = null;
|
||||
this.connected = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import type pino from "pino";
|
||||
|
||||
import type {
|
||||
SpeechToTextProvider,
|
||||
StreamingTranscriptionSession,
|
||||
TranscriptionResult,
|
||||
} from "../../../speech-provider.js";
|
||||
import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
|
||||
import {
|
||||
parsePcm16MonoWav,
|
||||
parsePcmRateFromFormat,
|
||||
pcm16lePeakAbs,
|
||||
pcm16leToFloat32,
|
||||
} from "../../../audio.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
|
||||
|
||||
export interface SherpaSttConfig {
|
||||
engine: SherpaOnlineRecognizerEngine;
|
||||
silencePeakThreshold?: number;
|
||||
tailPaddingMs?: number;
|
||||
}
|
||||
|
||||
export class SherpaOnnxSTT implements SpeechToTextProvider {
|
||||
private readonly engine: SherpaOnlineRecognizerEngine;
|
||||
private readonly silencePeakThreshold: number;
|
||||
private readonly tailPaddingMs: number;
|
||||
private readonly logger: pino.Logger;
|
||||
public readonly id = "local" as const;
|
||||
|
||||
constructor(config: SherpaSttConfig, logger: pino.Logger) {
|
||||
this.engine = config.engine;
|
||||
this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
|
||||
this.tailPaddingMs = config.tailPaddingMs ?? 500;
|
||||
this.logger = logger.child({ module: "speech", provider: "local", component: "stt" });
|
||||
}
|
||||
|
||||
public createSession(params: {
|
||||
logger: pino.Logger;
|
||||
language?: string;
|
||||
prompt?: string;
|
||||
}): StreamingTranscriptionSession {
|
||||
const emitter = new EventEmitter();
|
||||
void params;
|
||||
const requiredSampleRate = this.engine.sampleRate;
|
||||
let connected = false;
|
||||
let segmentId = uuidv4();
|
||||
let previousSegmentId: string | null = null;
|
||||
let pcm16: Buffer = Buffer.alloc(0);
|
||||
|
||||
return {
|
||||
requiredSampleRate,
|
||||
async connect() {
|
||||
connected = true;
|
||||
},
|
||||
appendPcm16(chunk: Buffer) {
|
||||
if (!connected) {
|
||||
emitter.emit("error", new Error("STT session not connected"));
|
||||
return;
|
||||
}
|
||||
pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]);
|
||||
},
|
||||
commit: () => {
|
||||
if (!connected) {
|
||||
emitter.emit("error", new Error("STT session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
const committedId = segmentId;
|
||||
const prev = previousSegmentId;
|
||||
emitter.emit("committed", { segmentId: committedId, previousSegmentId: prev });
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`);
|
||||
emitter.emit("transcript", {
|
||||
segmentId: committedId,
|
||||
transcript: rt.text,
|
||||
isFinal: true,
|
||||
language: rt.language,
|
||||
logprobs: rt.logprobs,
|
||||
avgLogprob: rt.avgLogprob,
|
||||
isLowConfidence: rt.isLowConfidence,
|
||||
});
|
||||
} catch (err) {
|
||||
emitter.emit("error", err);
|
||||
} finally {
|
||||
previousSegmentId = committedId;
|
||||
segmentId = uuidv4();
|
||||
pcm16 = Buffer.alloc(0);
|
||||
}
|
||||
})();
|
||||
},
|
||||
clear() {
|
||||
pcm16 = Buffer.alloc(0);
|
||||
segmentId = uuidv4();
|
||||
},
|
||||
close() {
|
||||
connected = false;
|
||||
pcm16 = Buffer.alloc(0);
|
||||
},
|
||||
on(event: "committed" | "transcript" | "error", handler: (payload: never) => void) {
|
||||
emitter.on(event, handler as (...args: unknown[]) => void);
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
public async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
|
||||
const start = Date.now();
|
||||
|
||||
let inputRate: number;
|
||||
let pcm16: Buffer;
|
||||
|
||||
if (format.toLowerCase().includes("audio/wav")) {
|
||||
const parsed = parsePcm16MonoWav(audioBuffer);
|
||||
inputRate = parsed.sampleRate;
|
||||
pcm16 = parsed.pcm16;
|
||||
} else if (format.toLowerCase().includes("audio/pcm")) {
|
||||
inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate;
|
||||
pcm16 = audioBuffer;
|
||||
} else {
|
||||
throw new Error(`Unsupported audio format for sherpa STT: ${format}`);
|
||||
}
|
||||
|
||||
const peak = pcm16lePeakAbs(pcm16);
|
||||
if (peak < this.silencePeakThreshold) {
|
||||
return { text: "", duration: Date.now() - start, isLowConfidence: true };
|
||||
}
|
||||
|
||||
let pcmForModel = pcm16;
|
||||
if (inputRate !== this.engine.sampleRate) {
|
||||
const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate });
|
||||
pcmForModel = resampler.processChunk(pcm16);
|
||||
inputRate = this.engine.sampleRate;
|
||||
}
|
||||
|
||||
const peakForModel = pcm16lePeakAbs(pcmForModel);
|
||||
const peakFloat = peakForModel / 32768.0;
|
||||
const targetPeak = 0.6;
|
||||
const maxGain = 50;
|
||||
const gain =
|
||||
peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
|
||||
|
||||
const stream = this.engine.createStream();
|
||||
try {
|
||||
const floatSamples = pcm16leToFloat32(pcmForModel, gain);
|
||||
stream.acceptWaveform(inputRate, floatSamples);
|
||||
|
||||
while (this.engine.recognizer.isReady(stream)) {
|
||||
this.engine.recognizer.decode(stream);
|
||||
}
|
||||
|
||||
const padSamples = Math.max(
|
||||
0,
|
||||
Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000),
|
||||
);
|
||||
if (padSamples > 0) {
|
||||
stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
|
||||
}
|
||||
|
||||
while (this.engine.recognizer.isReady(stream)) {
|
||||
this.engine.recognizer.decode(stream);
|
||||
}
|
||||
|
||||
const rawResult = this.engine.recognizer.getResult(stream);
|
||||
const text = (
|
||||
(typeof rawResult === "object" && rawResult && "text" in rawResult
|
||||
? rawResult.text
|
||||
: undefined) ?? ""
|
||||
).trim();
|
||||
const duration = Date.now() - start;
|
||||
this.logger.debug({ duration, textLength: text.length }, "Sherpa transcription complete");
|
||||
return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) };
|
||||
} finally {
|
||||
try {
|
||||
stream.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-p
|
||||
import { chunkBuffer, float32ToPcm16le } from "../../../audio.js";
|
||||
import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js";
|
||||
|
||||
export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16";
|
||||
export type SherpaTtsPreset = "kokoro-en-v0_19";
|
||||
|
||||
export interface SherpaTtsConfig {
|
||||
preset: SherpaTtsPreset;
|
||||
@@ -41,9 +41,6 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
constructor(config: SherpaTtsConfig, logger: pino.Logger) {
|
||||
if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
|
||||
throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
|
||||
}
|
||||
this.logger = logger.child({ module: "speech", provider: "local", component: "tts" });
|
||||
this.speakerId = config.speakerId ?? 0;
|
||||
this.speed = config.speed ?? 1.0;
|
||||
@@ -53,8 +50,7 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
||||
throw new Error("sherpa-onnx-node OfflineTts is unavailable");
|
||||
}
|
||||
|
||||
const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx";
|
||||
const modelPath = `${config.modelDir}/${modelFile}`;
|
||||
const modelPath = `${config.modelDir}/model.onnx`;
|
||||
const voicesPath = `${config.modelDir}/voices.bin`;
|
||||
const tokensPath = `${config.modelDir}/tokens.txt`;
|
||||
const dataDir = `${config.modelDir}/espeak-ng-data`;
|
||||
@@ -64,26 +60,15 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
||||
assertFileExists(tokensPath, "TTS tokens");
|
||||
assertFileExists(dataDir, "TTS espeak-ng dataDir");
|
||||
|
||||
const modelConfig =
|
||||
config.preset === "kokoro-en-v0_19"
|
||||
? {
|
||||
kokoro: {
|
||||
model: modelPath,
|
||||
voices: voicesPath,
|
||||
tokens: tokensPath,
|
||||
dataDir,
|
||||
lengthScale: config.lengthScale ?? 1.0,
|
||||
},
|
||||
}
|
||||
: {
|
||||
kitten: {
|
||||
model: modelPath,
|
||||
voices: voicesPath,
|
||||
tokens: tokensPath,
|
||||
dataDir,
|
||||
lengthScale: config.lengthScale ?? 1.0,
|
||||
},
|
||||
};
|
||||
const modelConfig = {
|
||||
kokoro: {
|
||||
model: modelPath,
|
||||
voices: voicesPath,
|
||||
tokens: tokensPath,
|
||||
dataDir,
|
||||
lengthScale: config.lengthScale ?? 1.0,
|
||||
},
|
||||
};
|
||||
|
||||
const offlineTtsConfig = {
|
||||
model: modelConfig,
|
||||
|
||||
+37
-107
@@ -5,28 +5,15 @@ import path from "node:path";
|
||||
import pino from "pino";
|
||||
|
||||
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
|
||||
import type { SherpaOnnxModelId } from "./model-catalog.js";
|
||||
import { createDaemonTestContext } from "../../../../test-utils/index.js";
|
||||
import { parsePcm16MonoWav, wordSimilarity } from "../../../../test-utils/dictation-e2e.js";
|
||||
import { SherpaOnnxTTS } from "./sherpa-tts.js";
|
||||
import { PocketTtsOnnxTTS } from "../pocket/pocket-tts-onnx.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
|
||||
import { SherpaOnnxSTT } from "./sherpa-stt.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
|
||||
import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js";
|
||||
|
||||
const RUN = process.env.PASEO_SPEECH_E2E_DOWNLOAD === "1";
|
||||
const downloadTest = RUN ? test : test.skip;
|
||||
|
||||
type ModelSet = "zipformer-kitten" | "parakeet-pocket";
|
||||
|
||||
function getModelSet(): ModelSet {
|
||||
const raw = (process.env.PASEO_SPEECH_E2E_MODEL_SET ?? "parakeet-pocket").trim().toLowerCase();
|
||||
if (raw === "zipformer-kitten" || raw === "zipformer") return "zipformer-kitten";
|
||||
if (raw === "parakeet-pocket" || raw === "parakeet") return "parakeet-pocket";
|
||||
throw new Error(`Unknown PASEO_SPEECH_E2E_MODEL_SET: ${raw}`);
|
||||
}
|
||||
|
||||
async function readFixtureWav(): Promise<Buffer> {
|
||||
const fixturePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.wav");
|
||||
return import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
|
||||
@@ -92,27 +79,20 @@ function toAudioPcmFormat(format: string): string {
|
||||
const rest = trimmed.replace(/^pcm;?/i, "");
|
||||
return rest ? `audio/pcm;${rest}` : "audio/pcm";
|
||||
}
|
||||
// Fall back to treating it as a suffix (e.g. "rate=24000")
|
||||
return `audio/pcm;${trimmed}`;
|
||||
}
|
||||
|
||||
downloadTest(
|
||||
"downloads models and runs streaming STT + streaming TTS",
|
||||
"downloads default local speech models and runs STT + TTS",
|
||||
async () => {
|
||||
const logger = pino({ level: "silent" });
|
||||
const set = getModelSet();
|
||||
|
||||
const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-"));
|
||||
const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech");
|
||||
|
||||
const modelIds: SherpaOnnxModelId[] =
|
||||
set === "parakeet-pocket"
|
||||
? ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"]
|
||||
: ["zipformer-bilingual-zh-en-2023-02-20", "kitten-nano-en-v0_1-fp16"];
|
||||
|
||||
await ensureSherpaOnnxModels({
|
||||
modelsDir,
|
||||
modelIds,
|
||||
modelIds: ["parakeet-tdt-0.6b-v2-int8", "kokoro-en-v0_19"],
|
||||
logger,
|
||||
});
|
||||
|
||||
@@ -128,16 +108,10 @@ downloadTest(
|
||||
local: {
|
||||
modelsDir,
|
||||
models: {
|
||||
dictationStt:
|
||||
set === "parakeet-pocket"
|
||||
? "parakeet-tdt-0.6b-v3-int8"
|
||||
: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceStt:
|
||||
set === "parakeet-pocket"
|
||||
? "parakeet-tdt-0.6b-v3-int8"
|
||||
: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceTts:
|
||||
set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
|
||||
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceTts: "kokoro-en-v0_19",
|
||||
voiceTtsSpeakerId: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -149,7 +123,6 @@ downloadTest(
|
||||
expect(sampleRate).toBe(16000);
|
||||
const format = "audio/pcm;rate=16000;bits=16";
|
||||
|
||||
// Streaming STT: dictation path (verifies websocket streaming + partials + final)
|
||||
const dictationId = `dict-download-${Date.now()}`;
|
||||
let partialCount = 0;
|
||||
const unsubscribe = ctx.client.on("dictation_stream_partial", (message) => {
|
||||
@@ -159,15 +132,14 @@ downloadTest(
|
||||
});
|
||||
|
||||
await ctx.client.startDictationStream(dictationId, format);
|
||||
const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
|
||||
const chunkBytes = 3200;
|
||||
let seq = 0;
|
||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||
ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
|
||||
seq += 1;
|
||||
}
|
||||
const finalSeq = seq - 1;
|
||||
const dictationFinal = await ctx.client.finishDictationStream(dictationId, finalSeq);
|
||||
const dictationFinal = await ctx.client.finishDictationStream(dictationId, seq - 1);
|
||||
unsubscribe();
|
||||
|
||||
expect(dictationFinal.text.toLowerCase()).toContain("voice note");
|
||||
@@ -175,7 +147,6 @@ downloadTest(
|
||||
expect(wordSimilarity(dictationFinal.text, baseline)).toBeGreaterThan(0.45);
|
||||
expect(partialCount).toBeGreaterThan(0);
|
||||
|
||||
// Voice-mode STT: chunked upload until isLast=true
|
||||
const transcriptionPromise = waitForSignal<string>(30000, (resolve, reject) => {
|
||||
const offResult = ctx.client.on("transcription_result", (message) => {
|
||||
if (message.type !== "transcription_result") return;
|
||||
@@ -218,80 +189,39 @@ downloadTest(
|
||||
await ctx.client.setVoiceMode(false);
|
||||
rmSync(voiceCwd, { recursive: true, force: true });
|
||||
|
||||
// Streaming TTS: generate locally from downloaded model and validate chunking.
|
||||
const ttsText = "This is a voice note.";
|
||||
if (set === "parakeet-pocket") {
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8");
|
||||
const tts = await PocketTtsOnnxTTS.create(
|
||||
{ modelDir, precision: "int8", targetChunkMs: 50 },
|
||||
logger,
|
||||
);
|
||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
||||
const { chunks, combined } = await readAllChunks(stream);
|
||||
const ttsModelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19");
|
||||
const tts = new SherpaOnnxTTS(
|
||||
{ preset: "kokoro-en-v0_19", modelDir: ttsModelDir, speakerId: 0 },
|
||||
logger,
|
||||
);
|
||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
||||
const { chunks, combined } = await readAllChunks(stream);
|
||||
tts.free();
|
||||
|
||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
||||
|
||||
// Round trip: TTS -> STT (offline parakeet)
|
||||
const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8");
|
||||
const engine = new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${sttModelDir}/encoder.int8.onnx`,
|
||||
decoder: `${sttModelDir}/decoder.int8.onnx`,
|
||||
joiner: `${sttModelDir}/joiner.int8.onnx`,
|
||||
tokens: `${sttModelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8");
|
||||
const engine = new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${sttModelDir}/encoder.int8.onnx`,
|
||||
decoder: `${sttModelDir}/decoder.int8.onnx`,
|
||||
joiner: `${sttModelDir}/joiner.int8.onnx`,
|
||||
tokens: `${sttModelDir}/tokens.txt`,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
const stt = new SherpaOnnxParakeetSTT({ engine }, logger);
|
||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
||||
engine.free();
|
||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
||||
} else {
|
||||
const ttsModelDir = path.join(modelsDir, "kitten-nano-en-v0_1-fp16");
|
||||
const tts = new SherpaOnnxTTS(
|
||||
{ preset: "kitten-nano-en-v0_1-fp16", modelDir: ttsModelDir },
|
||||
logger,
|
||||
);
|
||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
||||
const { chunks, combined } = await readAllChunks(stream);
|
||||
tts.free();
|
||||
|
||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
||||
|
||||
// Round trip: TTS -> STT (online zipformer, offline segment)
|
||||
const sttModelDir = path.join(
|
||||
modelsDir,
|
||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
);
|
||||
const engine = new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "transducer",
|
||||
encoder: `${sttModelDir}/encoder-epoch-99-avg-1.onnx`,
|
||||
decoder: `${sttModelDir}/decoder-epoch-99-avg-1.onnx`,
|
||||
joiner: `${sttModelDir}/joiner-epoch-99-avg-1.onnx`,
|
||||
tokens: `${sttModelDir}/tokens.txt`,
|
||||
modelType: "zipformer",
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
const stt = new SherpaOnnxSTT({ engine }, logger);
|
||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
||||
engine.free();
|
||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
||||
}
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
const stt = new SherpaOnnxParakeetSTT({ engine }, logger);
|
||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
||||
engine.free();
|
||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
||||
} finally {
|
||||
await ctx.cleanup();
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ describe("resolveSpeechConfig", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("resolves feature-scoped local model env vars", () => {
|
||||
test("resolves feature-scoped local speech settings", () => {
|
||||
const persisted = PersistedConfigSchema.parse({
|
||||
features: {
|
||||
voiceMode: {
|
||||
@@ -70,9 +70,9 @@ describe("resolveSpeechConfig", () => {
|
||||
},
|
||||
});
|
||||
const env = {
|
||||
PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer",
|
||||
PASEO_VOICE_LOCAL_STT_MODEL: "parakeet",
|
||||
PASEO_VOICE_LOCAL_TTS_MODEL: "kitten",
|
||||
PASEO_DICTATION_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
|
||||
PASEO_VOICE_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
|
||||
PASEO_VOICE_LOCAL_TTS_MODEL: "kokoro-en-v0_19",
|
||||
PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5",
|
||||
PASEO_VOICE_LOCAL_TTS_SPEED: "1.35",
|
||||
PASEO_DICTATION_LANGUAGE: "es",
|
||||
@@ -93,9 +93,9 @@ describe("resolveSpeechConfig", () => {
|
||||
expect(result.speech.local).toEqual({
|
||||
modelsDir: "/tmp/models",
|
||||
models: {
|
||||
dictationStt: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceStt: "parakeet-tdt-0.6b-v3-int8",
|
||||
voiceTts: "kitten-nano-en-v0_1-fp16",
|
||||
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceTts: "kokoro-en-v0_19",
|
||||
voiceTtsSpeakerId: 5,
|
||||
voiceTtsSpeed: 1.35,
|
||||
},
|
||||
@@ -120,9 +120,9 @@ describe("resolveSpeechConfig", () => {
|
||||
explicit: true,
|
||||
enabled: true,
|
||||
});
|
||||
expect(result.speech.local?.models.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20");
|
||||
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
|
||||
expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16");
|
||||
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||
expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
|
||||
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5);
|
||||
expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35);
|
||||
expect(result.speech.sttLanguages).toEqual({
|
||||
@@ -164,25 +164,6 @@ describe("resolveSpeechConfig", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("ignores deprecated shared local model env vars", () => {
|
||||
const persisted = PersistedConfigSchema.parse({});
|
||||
const env = {
|
||||
PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16",
|
||||
} as NodeJS.ProcessEnv;
|
||||
|
||||
const result = resolveSpeechConfig({
|
||||
paseoHome: "/tmp/paseo-home",
|
||||
env,
|
||||
persisted,
|
||||
});
|
||||
|
||||
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||
expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
|
||||
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0);
|
||||
});
|
||||
|
||||
test("respects disabled dictation and voice mode feature flags", () => {
|
||||
const persisted = PersistedConfigSchema.parse({
|
||||
features: {
|
||||
|
||||
@@ -293,8 +293,8 @@ function createReadySpeechReadinessSnapshot(): SpeechReadinessSnapshot {
|
||||
function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnapshot {
|
||||
return {
|
||||
generatedAt: "2026-02-14T00:00:00.000Z",
|
||||
requiredLocalModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
|
||||
missingLocalModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
|
||||
requiredLocalModelIds: ["parakeet-tdt-0.6b-v2-int8"],
|
||||
missingLocalModelIds: ["parakeet-tdt-0.6b-v2-int8"],
|
||||
download: {
|
||||
inProgress: true,
|
||||
error: null,
|
||||
@@ -320,9 +320,9 @@ function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnaps
|
||||
available: false,
|
||||
reasonCode: "model_download_in_progress",
|
||||
message:
|
||||
"Voice features are unavailable while models download in the background (sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20).",
|
||||
"Voice features are unavailable while models download in the background (parakeet-tdt-0.6b-v2-int8).",
|
||||
retryable: true,
|
||||
missingModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
|
||||
missingModelIds: ["parakeet-tdt-0.6b-v2-int8"],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user