diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 095a22e07..e0d4f3890 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -590,6 +590,7 @@ export async function createPaseoDaemon( command: voiceMcpBridgeCommand.command, baseArgs: [...voiceMcpBridgeCommand.baseArgs], env: { + ELECTRON_RUN_AS_NODE: "1", PASEO_HOME: config.paseoHome, }, }, diff --git a/packages/server/src/server/session.voice-mcp-config.test.ts b/packages/server/src/server/session.voice-mcp-config.test.ts index 9b5b7e1bf..98b1d4102 100644 --- a/packages/server/src/server/session.voice-mcp-config.test.ts +++ b/packages/server/src/server/session.voice-mcp-config.test.ts @@ -13,6 +13,7 @@ describe("voice MCP stdio config", () => { baseArgs: ["/tmp/mcp-stdio-socket-bridge-cli.mjs"], socketPath: "/tmp/paseo-voice.sock", env: { + ELECTRON_RUN_AS_NODE: "1", PASEO_HOME: "/tmp/paseo-home", }, }); @@ -25,6 +26,7 @@ describe("voice MCP stdio config", () => { "/tmp/paseo-voice.sock", ]); expect(config.env).toEqual({ + ELECTRON_RUN_AS_NODE: "1", PASEO_HOME: "/tmp/paseo-home", }); }); diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.test.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.test.ts new file mode 100644 index 000000000..582239206 --- /dev/null +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vitest"; +import pino from "pino"; + +import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js"; +import type { TranscriptionResult } from "../../../speech-provider.js"; + +function createDeferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +class TestSherpaOnnxParakeetStt extends SherpaOnnxParakeetSTT { + public readonly calls: Array<{ audio: Buffer; format: string }> = []; + public readonly pending: Array>> = []; + + constructor() { + super({ engine: { sampleRate: 16000 } as any }, pino({ level: "silent" })); + } + + override async transcribeAudio(audioBuffer: Buffer, format: string): Promise { + this.calls.push({ audio: Buffer.from(audioBuffer), format }); + const deferred = createDeferred(); + this.pending.push(deferred); + return deferred.promise; + } +} + +describe("SherpaOnnxParakeetSTT session", () => { + it("snapshots segment ids and buffers before async transcription starts", async () => { + const provider = new TestSherpaOnnxParakeetStt(); + const session = provider.createSession({ + logger: pino({ level: "silent" }), + language: "en", + }); + + const committed: Array<{ segmentId: string; previousSegmentId: string | null }> = []; + const transcripts: Array<{ segmentId: string; transcript: string; isFinal: boolean }> = []; + + session.on("committed", (payload) => { + committed.push(payload); + }); + session.on("transcript", (payload) => { + transcripts.push(payload); + }); + + await session.connect(); + + session.appendPcm16(Buffer.from([1, 2, 3, 4])); + session.commit(); + session.appendPcm16(Buffer.from([5, 6, 7, 8])); + session.commit(); + + expect(committed).toHaveLength(2); + expect(committed[1]?.segmentId).not.toBe(committed[0]?.segmentId); + expect(committed[0]?.previousSegmentId).toBeNull(); + expect(committed[1]?.previousSegmentId).toBe(committed[0]?.segmentId); + + expect(provider.calls).toEqual([ + { audio: Buffer.from([1, 2, 3, 4]), format: "audio/pcm;rate=16000" }, + { audio: Buffer.from([5, 6, 7, 8]), format: "audio/pcm;rate=16000" }, + ]); + + provider.pending[0]?.resolve({ text: "first", duration: 1 }); + provider.pending[1]?.resolve({ text: "second", duration: 1 }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(transcripts).toHaveLength(2); + expect(transcripts).toEqual([ + expect.objectContaining({ + segmentId: committed[0]!.segmentId, + transcript: "first", + isFinal: true, + }), + expect.objectContaining({ + segmentId: committed[1]!.segmentId, + transcript: "second", + isFinal: true, + }), + ]); + }); +}); diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts index 73552d14e..bba9f70c8 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts @@ -66,11 +66,18 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider { const committedId = segmentId; const prev = previousSegmentId; + const committedPcm16 = pcm16; + previousSegmentId = committedId; + segmentId = uuidv4(); + pcm16 = Buffer.alloc(0); (emitter as any).emit("committed", { segmentId: committedId, previousSegmentId: prev }); void (async () => { try { - const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`); + const rt = await this.transcribeAudio( + committedPcm16, + `audio/pcm;rate=${requiredSampleRate}`, + ); (emitter as any).emit("transcript", { segmentId: committedId, transcript: rt.text, @@ -83,10 +90,7 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider { } catch (err) { (emitter as any).emit("error", err); } finally { - previousSegmentId = committedId; - segmentId = uuidv4(); - pcm16 = Buffer.alloc(0); - logger.debug({ bytes: pcm16.length }, "Parakeet session reset"); + logger.debug({ bytes: committedPcm16.length }, "Parakeet session reset"); } })(); }, diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts new file mode 100644 index 000000000..c16929ec9 --- /dev/null +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import pino from "pino"; + +const generate = vi.fn(); +const free = vi.fn(); + +vi.mock("node:fs", () => ({ + existsSync: vi.fn(() => true), +})); + +vi.mock("./sherpa-onnx-node-loader.js", () => ({ + loadSherpaOnnxNode: () => ({ + OfflineTts: class { + public readonly sampleRate = 24000; + + constructor(_config: unknown) {} + + generate = generate; + free = free; + }, + }), +})); + +describe("SherpaOnnxTTS", () => { + beforeEach(() => { + generate.mockReset(); + free.mockReset(); + }); + + it("disables external buffers when calling sherpa generate", async () => { + generate.mockReturnValue({ + samples: Float32Array.from([0, 0.5, -0.5, 0.25]), + sampleRate: 24000, + }); + + const { SherpaOnnxTTS } = await import("./sherpa-tts.js"); + const tts = new SherpaOnnxTTS( + { + preset: "kokoro-en-v0_19", + modelDir: "/tmp/fake-model", + }, + pino({ level: "silent" }), + ); + + const result = await tts.synthesizeSpeech("hello"); + + expect(generate).toHaveBeenCalledWith({ + text: "hello", + sid: 0, + speed: 1, + enableExternalBuffer: false, + }); + expect(result.format).toBe("pcm;rate=24000"); + }); +}); diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts index 89627aee4..4a4c71a26 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts @@ -94,7 +94,14 @@ export class SherpaOnnxTTS implements TextToSpeechProvider { throw new Error("Cannot synthesize empty text"); } - const audio = this.tts.generate({ text: trimmed, sid: this.speakerId, speed: this.speed }); + const audio = this.tts.generate({ + text: trimmed, + sid: this.speakerId, + speed: this.speed, + // Electron rejects native external-backed typed arrays. Request a copied buffer + // from sherpa itself instead of trying to clone after generate() returns. + enableExternalBuffer: false, + }); const rawSamples: Float32Array | null = audio && audio.samples instanceof Float32Array ? audio.samples