fix(voice): harden Electron speak TTS path

This commit is contained in:
Mohamed Boudra
2026-04-05 00:53:50 +07:00
parent 5db9942125
commit bedf792616
6 changed files with 161 additions and 6 deletions
+1
View File
@@ -590,6 +590,7 @@ export async function createPaseoDaemon(
command: voiceMcpBridgeCommand.command,
baseArgs: [...voiceMcpBridgeCommand.baseArgs],
env: {
ELECTRON_RUN_AS_NODE: "1",
PASEO_HOME: config.paseoHome,
},
},
@@ -13,6 +13,7 @@ describe("voice MCP stdio config", () => {
baseArgs: ["/tmp/mcp-stdio-socket-bridge-cli.mjs"],
socketPath: "/tmp/paseo-voice.sock",
env: {
ELECTRON_RUN_AS_NODE: "1",
PASEO_HOME: "/tmp/paseo-home",
},
});
@@ -25,6 +26,7 @@ describe("voice MCP stdio config", () => {
"/tmp/paseo-voice.sock",
]);
expect(config.env).toEqual({
ELECTRON_RUN_AS_NODE: "1",
PASEO_HOME: "/tmp/paseo-home",
});
});
@@ -0,0 +1,86 @@
import { describe, expect, it } from "vitest";
import pino from "pino";
import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js";
import type { TranscriptionResult } from "../../../speech-provider.js";
function createDeferred<T>() {
let resolve!: (value: T | PromiseLike<T>) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}
class TestSherpaOnnxParakeetStt extends SherpaOnnxParakeetSTT {
public readonly calls: Array<{ audio: Buffer; format: string }> = [];
public readonly pending: Array<ReturnType<typeof createDeferred<TranscriptionResult>>> = [];
constructor() {
super({ engine: { sampleRate: 16000 } as any }, pino({ level: "silent" }));
}
override async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
this.calls.push({ audio: Buffer.from(audioBuffer), format });
const deferred = createDeferred<TranscriptionResult>();
this.pending.push(deferred);
return deferred.promise;
}
}
describe("SherpaOnnxParakeetSTT session", () => {
it("snapshots segment ids and buffers before async transcription starts", async () => {
const provider = new TestSherpaOnnxParakeetStt();
const session = provider.createSession({
logger: pino({ level: "silent" }),
language: "en",
});
const committed: Array<{ segmentId: string; previousSegmentId: string | null }> = [];
const transcripts: Array<{ segmentId: string; transcript: string; isFinal: boolean }> = [];
session.on("committed", (payload) => {
committed.push(payload);
});
session.on("transcript", (payload) => {
transcripts.push(payload);
});
await session.connect();
session.appendPcm16(Buffer.from([1, 2, 3, 4]));
session.commit();
session.appendPcm16(Buffer.from([5, 6, 7, 8]));
session.commit();
expect(committed).toHaveLength(2);
expect(committed[1]?.segmentId).not.toBe(committed[0]?.segmentId);
expect(committed[0]?.previousSegmentId).toBeNull();
expect(committed[1]?.previousSegmentId).toBe(committed[0]?.segmentId);
expect(provider.calls).toEqual([
{ audio: Buffer.from([1, 2, 3, 4]), format: "audio/pcm;rate=16000" },
{ audio: Buffer.from([5, 6, 7, 8]), format: "audio/pcm;rate=16000" },
]);
provider.pending[0]?.resolve({ text: "first", duration: 1 });
provider.pending[1]?.resolve({ text: "second", duration: 1 });
await new Promise((resolve) => setTimeout(resolve, 0));
expect(transcripts).toHaveLength(2);
expect(transcripts).toEqual([
expect.objectContaining({
segmentId: committed[0]!.segmentId,
transcript: "first",
isFinal: true,
}),
expect.objectContaining({
segmentId: committed[1]!.segmentId,
transcript: "second",
isFinal: true,
}),
]);
});
});
@@ -66,11 +66,18 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider {
const committedId = segmentId;
const prev = previousSegmentId;
const committedPcm16 = pcm16;
previousSegmentId = committedId;
segmentId = uuidv4();
pcm16 = Buffer.alloc(0);
(emitter as any).emit("committed", { segmentId: committedId, previousSegmentId: prev });
void (async () => {
try {
const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`);
const rt = await this.transcribeAudio(
committedPcm16,
`audio/pcm;rate=${requiredSampleRate}`,
);
(emitter as any).emit("transcript", {
segmentId: committedId,
transcript: rt.text,
@@ -83,10 +90,7 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider {
} catch (err) {
(emitter as any).emit("error", err);
} finally {
previousSegmentId = committedId;
segmentId = uuidv4();
pcm16 = Buffer.alloc(0);
logger.debug({ bytes: pcm16.length }, "Parakeet session reset");
logger.debug({ bytes: committedPcm16.length }, "Parakeet session reset");
}
})();
},
@@ -0,0 +1,55 @@
import { describe, expect, it, vi, beforeEach } from "vitest";
import pino from "pino";
const generate = vi.fn();
const free = vi.fn();
vi.mock("node:fs", () => ({
existsSync: vi.fn(() => true),
}));
vi.mock("./sherpa-onnx-node-loader.js", () => ({
loadSherpaOnnxNode: () => ({
OfflineTts: class {
public readonly sampleRate = 24000;
constructor(_config: unknown) {}
generate = generate;
free = free;
},
}),
}));
describe("SherpaOnnxTTS", () => {
beforeEach(() => {
generate.mockReset();
free.mockReset();
});
it("disables external buffers when calling sherpa generate", async () => {
generate.mockReturnValue({
samples: Float32Array.from([0, 0.5, -0.5, 0.25]),
sampleRate: 24000,
});
const { SherpaOnnxTTS } = await import("./sherpa-tts.js");
const tts = new SherpaOnnxTTS(
{
preset: "kokoro-en-v0_19",
modelDir: "/tmp/fake-model",
},
pino({ level: "silent" }),
);
const result = await tts.synthesizeSpeech("hello");
expect(generate).toHaveBeenCalledWith({
text: "hello",
sid: 0,
speed: 1,
enableExternalBuffer: false,
});
expect(result.format).toBe("pcm;rate=24000");
});
});
@@ -94,7 +94,14 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
throw new Error("Cannot synthesize empty text");
}
const audio = this.tts.generate({ text: trimmed, sid: this.speakerId, speed: this.speed });
const audio = this.tts.generate({
text: trimmed,
sid: this.speakerId,
speed: this.speed,
// Electron rejects native external-backed typed arrays. Request a copied buffer
// from sherpa itself instead of trying to clone after generate() returns.
enableExternalBuffer: false,
});
const rawSamples: Float32Array | null =
audio && audio.samples instanceof Float32Array
? audio.samples