fix(voice): harden Electron speak TTS path
This commit is contained in:
@@ -590,6 +590,7 @@ export async function createPaseoDaemon(
|
||||
command: voiceMcpBridgeCommand.command,
|
||||
baseArgs: [...voiceMcpBridgeCommand.baseArgs],
|
||||
env: {
|
||||
ELECTRON_RUN_AS_NODE: "1",
|
||||
PASEO_HOME: config.paseoHome,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -13,6 +13,7 @@ describe("voice MCP stdio config", () => {
|
||||
baseArgs: ["/tmp/mcp-stdio-socket-bridge-cli.mjs"],
|
||||
socketPath: "/tmp/paseo-voice.sock",
|
||||
env: {
|
||||
ELECTRON_RUN_AS_NODE: "1",
|
||||
PASEO_HOME: "/tmp/paseo-home",
|
||||
},
|
||||
});
|
||||
@@ -25,6 +26,7 @@ describe("voice MCP stdio config", () => {
|
||||
"/tmp/paseo-voice.sock",
|
||||
]);
|
||||
expect(config.env).toEqual({
|
||||
ELECTRON_RUN_AS_NODE: "1",
|
||||
PASEO_HOME: "/tmp/paseo-home",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import pino from "pino";
|
||||
|
||||
import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js";
|
||||
import type { TranscriptionResult } from "../../../speech-provider.js";
|
||||
|
||||
function createDeferred<T>() {
|
||||
let resolve!: (value: T | PromiseLike<T>) => void;
|
||||
let reject!: (reason?: unknown) => void;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
class TestSherpaOnnxParakeetStt extends SherpaOnnxParakeetSTT {
|
||||
public readonly calls: Array<{ audio: Buffer; format: string }> = [];
|
||||
public readonly pending: Array<ReturnType<typeof createDeferred<TranscriptionResult>>> = [];
|
||||
|
||||
constructor() {
|
||||
super({ engine: { sampleRate: 16000 } as any }, pino({ level: "silent" }));
|
||||
}
|
||||
|
||||
override async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
|
||||
this.calls.push({ audio: Buffer.from(audioBuffer), format });
|
||||
const deferred = createDeferred<TranscriptionResult>();
|
||||
this.pending.push(deferred);
|
||||
return deferred.promise;
|
||||
}
|
||||
}
|
||||
|
||||
describe("SherpaOnnxParakeetSTT session", () => {
|
||||
it("snapshots segment ids and buffers before async transcription starts", async () => {
|
||||
const provider = new TestSherpaOnnxParakeetStt();
|
||||
const session = provider.createSession({
|
||||
logger: pino({ level: "silent" }),
|
||||
language: "en",
|
||||
});
|
||||
|
||||
const committed: Array<{ segmentId: string; previousSegmentId: string | null }> = [];
|
||||
const transcripts: Array<{ segmentId: string; transcript: string; isFinal: boolean }> = [];
|
||||
|
||||
session.on("committed", (payload) => {
|
||||
committed.push(payload);
|
||||
});
|
||||
session.on("transcript", (payload) => {
|
||||
transcripts.push(payload);
|
||||
});
|
||||
|
||||
await session.connect();
|
||||
|
||||
session.appendPcm16(Buffer.from([1, 2, 3, 4]));
|
||||
session.commit();
|
||||
session.appendPcm16(Buffer.from([5, 6, 7, 8]));
|
||||
session.commit();
|
||||
|
||||
expect(committed).toHaveLength(2);
|
||||
expect(committed[1]?.segmentId).not.toBe(committed[0]?.segmentId);
|
||||
expect(committed[0]?.previousSegmentId).toBeNull();
|
||||
expect(committed[1]?.previousSegmentId).toBe(committed[0]?.segmentId);
|
||||
|
||||
expect(provider.calls).toEqual([
|
||||
{ audio: Buffer.from([1, 2, 3, 4]), format: "audio/pcm;rate=16000" },
|
||||
{ audio: Buffer.from([5, 6, 7, 8]), format: "audio/pcm;rate=16000" },
|
||||
]);
|
||||
|
||||
provider.pending[0]?.resolve({ text: "first", duration: 1 });
|
||||
provider.pending[1]?.resolve({ text: "second", duration: 1 });
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
expect(transcripts).toHaveLength(2);
|
||||
expect(transcripts).toEqual([
|
||||
expect.objectContaining({
|
||||
segmentId: committed[0]!.segmentId,
|
||||
transcript: "first",
|
||||
isFinal: true,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
segmentId: committed[1]!.segmentId,
|
||||
transcript: "second",
|
||||
isFinal: true,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -66,11 +66,18 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider {
|
||||
|
||||
const committedId = segmentId;
|
||||
const prev = previousSegmentId;
|
||||
const committedPcm16 = pcm16;
|
||||
previousSegmentId = committedId;
|
||||
segmentId = uuidv4();
|
||||
pcm16 = Buffer.alloc(0);
|
||||
(emitter as any).emit("committed", { segmentId: committedId, previousSegmentId: prev });
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`);
|
||||
const rt = await this.transcribeAudio(
|
||||
committedPcm16,
|
||||
`audio/pcm;rate=${requiredSampleRate}`,
|
||||
);
|
||||
(emitter as any).emit("transcript", {
|
||||
segmentId: committedId,
|
||||
transcript: rt.text,
|
||||
@@ -83,10 +90,7 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider {
|
||||
} catch (err) {
|
||||
(emitter as any).emit("error", err);
|
||||
} finally {
|
||||
previousSegmentId = committedId;
|
||||
segmentId = uuidv4();
|
||||
pcm16 = Buffer.alloc(0);
|
||||
logger.debug({ bytes: pcm16.length }, "Parakeet session reset");
|
||||
logger.debug({ bytes: committedPcm16.length }, "Parakeet session reset");
|
||||
}
|
||||
})();
|
||||
},
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
import pino from "pino";
|
||||
|
||||
const generate = vi.fn();
|
||||
const free = vi.fn();
|
||||
|
||||
vi.mock("node:fs", () => ({
|
||||
existsSync: vi.fn(() => true),
|
||||
}));
|
||||
|
||||
vi.mock("./sherpa-onnx-node-loader.js", () => ({
|
||||
loadSherpaOnnxNode: () => ({
|
||||
OfflineTts: class {
|
||||
public readonly sampleRate = 24000;
|
||||
|
||||
constructor(_config: unknown) {}
|
||||
|
||||
generate = generate;
|
||||
free = free;
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
describe("SherpaOnnxTTS", () => {
|
||||
beforeEach(() => {
|
||||
generate.mockReset();
|
||||
free.mockReset();
|
||||
});
|
||||
|
||||
it("disables external buffers when calling sherpa generate", async () => {
|
||||
generate.mockReturnValue({
|
||||
samples: Float32Array.from([0, 0.5, -0.5, 0.25]),
|
||||
sampleRate: 24000,
|
||||
});
|
||||
|
||||
const { SherpaOnnxTTS } = await import("./sherpa-tts.js");
|
||||
const tts = new SherpaOnnxTTS(
|
||||
{
|
||||
preset: "kokoro-en-v0_19",
|
||||
modelDir: "/tmp/fake-model",
|
||||
},
|
||||
pino({ level: "silent" }),
|
||||
);
|
||||
|
||||
const result = await tts.synthesizeSpeech("hello");
|
||||
|
||||
expect(generate).toHaveBeenCalledWith({
|
||||
text: "hello",
|
||||
sid: 0,
|
||||
speed: 1,
|
||||
enableExternalBuffer: false,
|
||||
});
|
||||
expect(result.format).toBe("pcm;rate=24000");
|
||||
});
|
||||
});
|
||||
@@ -94,7 +94,14 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
||||
throw new Error("Cannot synthesize empty text");
|
||||
}
|
||||
|
||||
const audio = this.tts.generate({ text: trimmed, sid: this.speakerId, speed: this.speed });
|
||||
const audio = this.tts.generate({
|
||||
text: trimmed,
|
||||
sid: this.speakerId,
|
||||
speed: this.speed,
|
||||
// Electron rejects native external-backed typed arrays. Request a copied buffer
|
||||
// from sherpa itself instead of trying to clone after generate() returns.
|
||||
enableExternalBuffer: false,
|
||||
});
|
||||
const rawSamples: Float32Array | null =
|
||||
audio && audio.samples instanceof Float32Array
|
||||
? audio.samples
|
||||
|
||||
Reference in New Issue
Block a user