Simplify local speech models
This commit is contained in:
Mohamed Boudra
@@ -171,6 +171,8 @@ Single file, validated with `PersistedConfigSchema`.
|
|||||||
|
|
||||||
All fields are optional with sensible defaults.
|
All fields are optional with sensible defaults.
|
||||||
|
|
||||||
|
Local speech model ids are intentionally narrow: STT uses `parakeet-tdt-0.6b-v2-int8`, TTS uses `kokoro-en-v0_19`, and turn detection uses the bundled Silero VAD model.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. Schedule
|
## 3. Schedule
|
||||||
|
|||||||
Generated
+29
-66
@@ -10647,16 +10647,6 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/@sctg/sentencepiece-js": {
|
|
||||||
"version": "1.3.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/@sctg/sentencepiece-js/-/sentencepiece-js-1.3.3.tgz",
|
|
||||||
"integrity": "sha512-iPnzR2HGjdQQG2SpTyPH3wnnpgQ2aS14B3I2jHjomkoaMprM3Sn+WOQblHwByetnvn3n14y3IwjwogVUHA2cVA==",
|
|
||||||
"license": "Apache-2.0",
|
|
||||||
"dependencies": {
|
|
||||||
"app-root-path": "^3.1.0",
|
|
||||||
"buffer": "^6.0.3"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@segment/ajv-human-errors": {
|
"node_modules/@segment/ajv-human-errors": {
|
||||||
"version": "2.16.0",
|
"version": "2.16.0",
|
||||||
"resolved": "https://registry.npmjs.org/@segment/ajv-human-errors/-/ajv-human-errors-2.16.0.tgz",
|
"resolved": "https://registry.npmjs.org/@segment/ajv-human-errors/-/ajv-human-errors-2.16.0.tgz",
|
||||||
@@ -13823,15 +13813,6 @@
|
|||||||
"node": ">=0.4.0"
|
"node": ">=0.4.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/adm-zip": {
|
|
||||||
"version": "0.5.16",
|
|
||||||
"resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz",
|
|
||||||
"integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==",
|
|
||||||
"license": "MIT",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/agent-base": {
|
"node_modules/agent-base": {
|
||||||
"version": "6.0.2",
|
"version": "6.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
|
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
|
||||||
@@ -14283,15 +14264,6 @@
|
|||||||
"node": ">=18"
|
"node": ">=18"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/app-root-path": {
|
|
||||||
"version": "3.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz",
|
|
||||||
"integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==",
|
|
||||||
"license": "MIT",
|
|
||||||
"engines": {
|
|
||||||
"node": ">= 6.0.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/arg": {
|
"node_modules/arg": {
|
||||||
"version": "5.0.2",
|
"version": "5.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
|
"resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
|
||||||
@@ -15147,7 +15119,9 @@
|
|||||||
"resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz",
|
||||||
"integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==",
|
"integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==",
|
||||||
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
|
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
|
||||||
"license": "MIT"
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true
|
||||||
},
|
},
|
||||||
"node_modules/bplist-creator": {
|
"node_modules/bplist-creator": {
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
@@ -17013,7 +16987,9 @@
|
|||||||
"version": "2.1.0",
|
"version": "2.1.0",
|
||||||
"resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
|
||||||
"integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==",
|
"integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==",
|
||||||
"license": "MIT"
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true
|
||||||
},
|
},
|
||||||
"node_modules/detect-node-es": {
|
"node_modules/detect-node-es": {
|
||||||
"version": "1.1.0",
|
"version": "1.1.0",
|
||||||
@@ -18115,7 +18091,9 @@
|
|||||||
"version": "4.1.1",
|
"version": "4.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz",
|
||||||
"integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==",
|
"integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==",
|
||||||
"license": "MIT"
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true
|
||||||
},
|
},
|
||||||
"node_modules/esbuild": {
|
"node_modules/esbuild": {
|
||||||
"version": "0.27.3",
|
"version": "0.27.3",
|
||||||
@@ -22620,7 +22598,9 @@
|
|||||||
"version": "3.0.0",
|
"version": "3.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
|
||||||
"integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==",
|
"integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==",
|
||||||
|
"dev": true,
|
||||||
"license": "BSD-3-Clause",
|
"license": "BSD-3-Clause",
|
||||||
|
"optional": true,
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"boolean": "^3.0.1",
|
"boolean": "^3.0.1",
|
||||||
"es6-error": "^4.1.1",
|
"es6-error": "^4.1.1",
|
||||||
@@ -22637,7 +22617,9 @@
|
|||||||
"version": "7.0.1",
|
"version": "7.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz",
|
||||||
"integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==",
|
"integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==",
|
||||||
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"type-fest": "^0.13.1"
|
"type-fest": "^0.13.1"
|
||||||
},
|
},
|
||||||
@@ -22652,7 +22634,9 @@
|
|||||||
"version": "0.13.1",
|
"version": "0.13.1",
|
||||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
|
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
|
||||||
"integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==",
|
"integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==",
|
||||||
|
"dev": true,
|
||||||
"license": "(MIT OR CC0-1.0)",
|
"license": "(MIT OR CC0-1.0)",
|
||||||
|
"optional": true,
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=10"
|
"node": ">=10"
|
||||||
},
|
},
|
||||||
@@ -22677,6 +22661,7 @@
|
|||||||
"version": "1.0.4",
|
"version": "1.0.4",
|
||||||
"resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz",
|
"resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz",
|
||||||
"integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==",
|
"integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==",
|
||||||
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"define-properties": "^1.2.1",
|
"define-properties": "^1.2.1",
|
||||||
@@ -25331,7 +25316,9 @@
|
|||||||
"version": "5.0.1",
|
"version": "5.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
|
||||||
"integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
|
"integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
|
||||||
"license": "ISC"
|
"dev": true,
|
||||||
|
"license": "ISC",
|
||||||
|
"optional": true
|
||||||
},
|
},
|
||||||
"node_modules/json5": {
|
"node_modules/json5": {
|
||||||
"version": "2.2.3",
|
"version": "2.2.3",
|
||||||
@@ -26349,7 +26336,9 @@
|
|||||||
"version": "3.0.0",
|
"version": "3.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
|
||||||
"integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
|
"integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
|
||||||
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"escape-string-regexp": "^4.0.0"
|
"escape-string-regexp": "^4.0.0"
|
||||||
},
|
},
|
||||||
@@ -28985,29 +28974,6 @@
|
|||||||
"regex-recursion": "^6.0.2"
|
"regex-recursion": "^6.0.2"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/onnxruntime-common": {
|
|
||||||
"version": "1.24.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz",
|
|
||||||
"integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==",
|
|
||||||
"license": "MIT"
|
|
||||||
},
|
|
||||||
"node_modules/onnxruntime-node": {
|
|
||||||
"version": "1.24.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz",
|
|
||||||
"integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==",
|
|
||||||
"hasInstallScript": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"os": [
|
|
||||||
"win32",
|
|
||||||
"darwin",
|
|
||||||
"linux"
|
|
||||||
],
|
|
||||||
"dependencies": {
|
|
||||||
"adm-zip": "^0.5.16",
|
|
||||||
"global-agent": "^3.0.0",
|
|
||||||
"onnxruntime-common": "1.24.3"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/open": {
|
"node_modules/open": {
|
||||||
"version": "7.4.2",
|
"version": "7.4.2",
|
||||||
"resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz",
|
"resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz",
|
||||||
@@ -31922,7 +31888,9 @@
|
|||||||
"version": "2.15.4",
|
"version": "2.15.4",
|
||||||
"resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
|
"resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
|
||||||
"integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==",
|
"integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==",
|
||||||
|
"dev": true,
|
||||||
"license": "BSD-3-Clause",
|
"license": "BSD-3-Clause",
|
||||||
|
"optional": true,
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"boolean": "^3.0.1",
|
"boolean": "^3.0.1",
|
||||||
"detect-node": "^2.0.4",
|
"detect-node": "^2.0.4",
|
||||||
@@ -31939,7 +31907,9 @@
|
|||||||
"version": "1.1.3",
|
"version": "1.1.3",
|
||||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
|
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
|
||||||
"integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
|
"integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
|
||||||
"license": "BSD-3-Clause"
|
"dev": true,
|
||||||
|
"license": "BSD-3-Clause",
|
||||||
|
"optional": true
|
||||||
},
|
},
|
||||||
"node_modules/rollup": {
|
"node_modules/rollup": {
|
||||||
"version": "4.59.0",
|
"version": "4.59.0",
|
||||||
@@ -32230,7 +32200,9 @@
|
|||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz",
|
||||||
"integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==",
|
"integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==",
|
||||||
"license": "MIT"
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true
|
||||||
},
|
},
|
||||||
"node_modules/send": {
|
"node_modules/send": {
|
||||||
"version": "0.19.2",
|
"version": "0.19.2",
|
||||||
@@ -32515,12 +32487,6 @@
|
|||||||
"url": "https://github.com/sponsors/ljharb"
|
"url": "https://github.com/sponsors/ljharb"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/sherpa-onnx": {
|
|
||||||
"version": "1.12.28",
|
|
||||||
"resolved": "https://registry.npmjs.org/sherpa-onnx/-/sherpa-onnx-1.12.28.tgz",
|
|
||||||
"integrity": "sha512-PaOuTCUA/Vh5o3zcq+z3j1Zpv2VlP7iubOgYoVODDyeFDicRxjskiEQqERdWEcpQUi6vh47q3A63bS7Y9FRHNg==",
|
|
||||||
"license": "Apache-2.0"
|
|
||||||
},
|
|
||||||
"node_modules/sherpa-onnx-darwin-arm64": {
|
"node_modules/sherpa-onnx-darwin-arm64": {
|
||||||
"version": "1.12.28",
|
"version": "1.12.28",
|
||||||
"resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.12.28.tgz",
|
"resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.12.28.tgz",
|
||||||
@@ -37603,7 +37569,6 @@
|
|||||||
"@isaacs/ttlcache": "^2.1.4",
|
"@isaacs/ttlcache": "^2.1.4",
|
||||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||||
"@opencode-ai/sdk": "1.14.46",
|
"@opencode-ai/sdk": "1.14.46",
|
||||||
"@sctg/sentencepiece-js": "^1.1.0",
|
|
||||||
"@xterm/headless": "^6.0.0",
|
"@xterm/headless": "^6.0.0",
|
||||||
"ai": "5.0.78",
|
"ai": "5.0.78",
|
||||||
"ajv": "^8.20.0",
|
"ajv": "^8.20.0",
|
||||||
@@ -37614,7 +37579,6 @@
|
|||||||
"lru-cache": "^11.5.1",
|
"lru-cache": "^11.5.1",
|
||||||
"mnemonic-id": "^3.2.7",
|
"mnemonic-id": "^3.2.7",
|
||||||
"node-pty": "1.2.0-beta.11",
|
"node-pty": "1.2.0-beta.11",
|
||||||
"onnxruntime-node": "^1.23.0",
|
|
||||||
"openai": "^4.20.0",
|
"openai": "^4.20.0",
|
||||||
"p-limit": "^7.3.0",
|
"p-limit": "^7.3.0",
|
||||||
"p-memoize": "^8.0.0",
|
"p-memoize": "^8.0.0",
|
||||||
@@ -37622,7 +37586,6 @@
|
|||||||
"pino-pretty": "^13.1.3",
|
"pino-pretty": "^13.1.3",
|
||||||
"qrcode": "^1.5.4",
|
"qrcode": "^1.5.4",
|
||||||
"rotating-file-stream": "^3.2.9",
|
"rotating-file-stream": "^3.2.9",
|
||||||
"sherpa-onnx": "1.12.28",
|
|
||||||
"sherpa-onnx-node": "1.12.28",
|
"sherpa-onnx-node": "1.12.28",
|
||||||
"strip-ansi": "^7.1.2",
|
"strip-ansi": "^7.1.2",
|
||||||
"tree-kill": "^1.2.2",
|
"tree-kill": "^1.2.2",
|
||||||
|
|||||||
@@ -27,29 +27,6 @@ function pruneChildrenExcept(parent, keep) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function pruneOnnxRuntime(nodeModules, platform, arch) {
|
|
||||||
const onnxBin = path.join(nodeModules, "onnxruntime-node", "bin", "napi-v6");
|
|
||||||
if (!fs.existsSync(onnxBin)) return;
|
|
||||||
|
|
||||||
const otherPlatforms = ["darwin", "linux", "win32"].filter((p) => p !== platform);
|
|
||||||
for (const p of otherPlatforms) {
|
|
||||||
rmSafe(path.join(onnxBin, p));
|
|
||||||
}
|
|
||||||
|
|
||||||
pruneChildrenExcept(path.join(onnxBin, platform), new Set([arch]));
|
|
||||||
|
|
||||||
if (platform === "linux") {
|
|
||||||
const archDir = path.join(onnxBin, "linux", arch);
|
|
||||||
if (fs.existsSync(archDir)) {
|
|
||||||
for (const name of fs.readdirSync(archDir)) {
|
|
||||||
if (name.includes("cuda") || name.includes("tensorrt")) {
|
|
||||||
fs.rmSync(path.join(archDir, name), { force: true });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function pruneClaudeAgentSdk(nodeModules, platform, arch) {
|
function pruneClaudeAgentSdk(nodeModules, platform, arch) {
|
||||||
const vendorRoot = path.join(nodeModules, "@anthropic-ai", "claude-agent-sdk", "vendor");
|
const vendorRoot = path.join(nodeModules, "@anthropic-ai", "claude-agent-sdk", "vendor");
|
||||||
const keepName = RIPGREP_PLATFORM_DIR[platform]?.[arch];
|
const keepName = RIPGREP_PLATFORM_DIR[platform]?.[arch];
|
||||||
@@ -107,7 +84,6 @@ function pruneNativeModules(appOutDir, platform, arch) {
|
|||||||
|
|
||||||
const before = dirSizeSync(nodeModules);
|
const before = dirSizeSync(nodeModules);
|
||||||
|
|
||||||
pruneOnnxRuntime(nodeModules, platform, arch);
|
|
||||||
pruneClaudeAgentSdk(nodeModules, platform, arch);
|
pruneClaudeAgentSdk(nodeModules, platform, arch);
|
||||||
pruneNodePty(nodeModules, platform, arch);
|
pruneNodePty(nodeModules, platform, arch);
|
||||||
pruneSharpLibvips(nodeModules, platform, arch);
|
pruneSharpLibvips(nodeModules, platform, arch);
|
||||||
|
|||||||
@@ -44,7 +44,6 @@
|
|||||||
"generate:config-schema": "tsx scripts/generate-config-schema.ts",
|
"generate:config-schema": "tsx scripts/generate-config-schema.ts",
|
||||||
"speech:models": "tsx scripts/list-speech-models.ts",
|
"speech:models": "tsx scripts/list-speech-models.ts",
|
||||||
"speech:download": "tsx scripts/download-speech-models.ts",
|
"speech:download": "tsx scripts/download-speech-models.ts",
|
||||||
"speech:tts:matrix": "tsx scripts/generate-sherpa-tts-matrix.ts",
|
|
||||||
"speech:transcribe:local": "tsx scripts/transcribe-local-wav.ts",
|
"speech:transcribe:local": "tsx scripts/transcribe-local-wav.ts",
|
||||||
"test": "npm run test:unit && npm run test:integration",
|
"test": "npm run test:unit && npm run test:integration",
|
||||||
"test:unit": "vitest run --exclude \"**/*.e2e.test.ts\"",
|
"test:unit": "vitest run --exclude \"**/*.e2e.test.ts\"",
|
||||||
@@ -70,7 +69,6 @@
|
|||||||
"@isaacs/ttlcache": "^2.1.4",
|
"@isaacs/ttlcache": "^2.1.4",
|
||||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||||
"@opencode-ai/sdk": "1.14.46",
|
"@opencode-ai/sdk": "1.14.46",
|
||||||
"@sctg/sentencepiece-js": "^1.1.0",
|
|
||||||
"@xterm/headless": "^6.0.0",
|
"@xterm/headless": "^6.0.0",
|
||||||
"ai": "5.0.78",
|
"ai": "5.0.78",
|
||||||
"ajv": "^8.20.0",
|
"ajv": "^8.20.0",
|
||||||
@@ -81,7 +79,6 @@
|
|||||||
"lru-cache": "^11.5.1",
|
"lru-cache": "^11.5.1",
|
||||||
"mnemonic-id": "^3.2.7",
|
"mnemonic-id": "^3.2.7",
|
||||||
"node-pty": "1.2.0-beta.11",
|
"node-pty": "1.2.0-beta.11",
|
||||||
"onnxruntime-node": "^1.23.0",
|
|
||||||
"openai": "^4.20.0",
|
"openai": "^4.20.0",
|
||||||
"p-limit": "^7.3.0",
|
"p-limit": "^7.3.0",
|
||||||
"p-memoize": "^8.0.0",
|
"p-memoize": "^8.0.0",
|
||||||
@@ -89,7 +86,6 @@
|
|||||||
"pino-pretty": "^13.1.3",
|
"pino-pretty": "^13.1.3",
|
||||||
"qrcode": "^1.5.4",
|
"qrcode": "^1.5.4",
|
||||||
"rotating-file-stream": "^3.2.9",
|
"rotating-file-stream": "^3.2.9",
|
||||||
"sherpa-onnx": "1.12.28",
|
|
||||||
"sherpa-onnx-node": "1.12.28",
|
"sherpa-onnx-node": "1.12.28",
|
||||||
"strip-ansi": "^7.1.2",
|
"strip-ansi": "^7.1.2",
|
||||||
"tree-kill": "^1.2.2",
|
"tree-kill": "^1.2.2",
|
||||||
|
|||||||
@@ -31,11 +31,7 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: LocalSpeechMo
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (modelIds.length === 0) {
|
if (modelIds.length === 0) {
|
||||||
const stt = (process.env.PASEO_LOCAL_STT_MODEL ||
|
modelIds.push(DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL);
|
||||||
DEFAULT_LOCAL_STT_MODEL) as LocalSpeechModelId;
|
|
||||||
const tts = (process.env.PASEO_LOCAL_TTS_MODEL ||
|
|
||||||
DEFAULT_LOCAL_TTS_MODEL) as LocalSpeechModelId;
|
|
||||||
modelIds.push(stt, tts);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return { modelsDir, modelIds };
|
return { modelsDir, modelIds };
|
||||||
|
|||||||
@@ -1,436 +0,0 @@
|
|||||||
import { mkdir, writeFile } from "node:fs/promises";
|
|
||||||
import path from "node:path";
|
|
||||||
|
|
||||||
import { createRootLogger } from "../src/server/logger.js";
|
|
||||||
import { resolvePaseoHome } from "../src/server/paseo-home.js";
|
|
||||||
import { float32ToPcm16le, parsePcmRateFromFormat } from "../src/server/speech/audio.js";
|
|
||||||
import {
|
|
||||||
ensureLocalSpeechModels,
|
|
||||||
getLocalSpeechModelDir,
|
|
||||||
type LocalTtsModelId,
|
|
||||||
} from "../src/server/speech/providers/local/models.js";
|
|
||||||
import { PocketTtsOnnxTTS } from "../src/server/speech/providers/local/pocket/pocket-tts-onnx.js";
|
|
||||||
import {
|
|
||||||
LOCAL_TTS_MODEL_IDS,
|
|
||||||
LocalTtsModelIdSchema,
|
|
||||||
} from "../src/server/speech/providers/local/sherpa/model-catalog.js";
|
|
||||||
import { loadSherpaOnnxNode } from "../src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.js";
|
|
||||||
|
|
||||||
const DEFAULT_SAMPLE_TEXT =
|
|
||||||
"Paseo keeps your coding agents in your pocket, with fast local speech and reliable remote control.";
|
|
||||||
|
|
||||||
interface ScriptOptions {
|
|
||||||
modelsDir: string;
|
|
||||||
outputDir: string;
|
|
||||||
text: string;
|
|
||||||
speed: number;
|
|
||||||
modelIds: LocalTtsModelId[];
|
|
||||||
}
|
|
||||||
|
|
||||||
interface GeneratedSample {
|
|
||||||
modelId: LocalTtsModelId;
|
|
||||||
voiceLabel: string;
|
|
||||||
speakerId: number | null;
|
|
||||||
sampleRate: number;
|
|
||||||
durationSeconds: number;
|
|
||||||
relativePath: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
function usage(): string {
|
|
||||||
return [
|
|
||||||
"Generate a local Sherpa/Pocket TTS sample matrix.",
|
|
||||||
"",
|
|
||||||
"Options:",
|
|
||||||
" --models-dir <dir> Directory for downloaded local speech models",
|
|
||||||
" --output-dir <dir> Directory where WAV samples are written",
|
|
||||||
" --model <id> TTS model to include (repeatable)",
|
|
||||||
" --text <sentence> Sample sentence used for every generated file",
|
|
||||||
" --speed <number> TTS speed multiplier for Sherpa models (default: 1.0)",
|
|
||||||
" --help Show this help",
|
|
||||||
"",
|
|
||||||
`Known TTS model IDs: ${LOCAL_TTS_MODEL_IDS.join(", ")}`,
|
|
||||||
].join("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
function parsePositiveNumber(raw: string, flag: string): number {
|
|
||||||
const parsed = Number.parseFloat(raw);
|
|
||||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
||||||
throw new Error(`${flag} must be a positive number (received: ${raw})`);
|
|
||||||
}
|
|
||||||
return parsed;
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseArgs(argv: string[]): ScriptOptions {
|
|
||||||
const paseoHome = resolvePaseoHome();
|
|
||||||
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
||||||
|
|
||||||
let modelsDir = process.env.PASEO_LOCAL_MODELS_DIR ?? `${paseoHome}/models/local-speech`;
|
|
||||||
let outputDir = path.resolve(process.cwd(), ".debug", "sherpa-tts-matrix", timestamp);
|
|
||||||
let text = DEFAULT_SAMPLE_TEXT;
|
|
||||||
let speed = parsePositiveNumber(process.env.PASEO_VOICE_LOCAL_TTS_SPEED ?? "1.0", "--speed");
|
|
||||||
const requestedModelIds: LocalTtsModelId[] = [];
|
|
||||||
|
|
||||||
for (let i = 0; i < argv.length; i += 1) {
|
|
||||||
const arg = argv[i];
|
|
||||||
|
|
||||||
if (arg === "--help") {
|
|
||||||
// eslint-disable-next-line no-console
|
|
||||||
console.log(usage());
|
|
||||||
process.exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (arg === "--models-dir") {
|
|
||||||
modelsDir = path.resolve(argv[i + 1] ?? modelsDir);
|
|
||||||
i += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (arg === "--output-dir") {
|
|
||||||
outputDir = path.resolve(argv[i + 1] ?? outputDir);
|
|
||||||
i += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (arg === "--text") {
|
|
||||||
text = argv[i + 1] ?? text;
|
|
||||||
i += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (arg === "--speed") {
|
|
||||||
const raw = argv[i + 1];
|
|
||||||
if (!raw) {
|
|
||||||
throw new Error("--speed requires a value");
|
|
||||||
}
|
|
||||||
speed = parsePositiveNumber(raw, "--speed");
|
|
||||||
i += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (arg === "--model") {
|
|
||||||
const raw = argv[i + 1];
|
|
||||||
if (!raw) {
|
|
||||||
throw new Error("--model requires a value");
|
|
||||||
}
|
|
||||||
requestedModelIds.push(LocalTtsModelIdSchema.parse(raw));
|
|
||||||
i += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(`Unknown argument: ${arg}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const trimmed = text.trim();
|
|
||||||
if (!trimmed) {
|
|
||||||
throw new Error("Sample text cannot be empty");
|
|
||||||
}
|
|
||||||
|
|
||||||
const modelIds =
|
|
||||||
requestedModelIds.length > 0
|
|
||||||
? Array.from(new Set(requestedModelIds))
|
|
||||||
: [...LOCAL_TTS_MODEL_IDS];
|
|
||||||
|
|
||||||
return {
|
|
||||||
modelsDir,
|
|
||||||
outputDir,
|
|
||||||
text: trimmed,
|
|
||||||
speed,
|
|
||||||
modelIds,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function pcm16MonoToWavBuffer(pcm16: Buffer, sampleRate: number): Buffer {
|
|
||||||
const channels = 1;
|
|
||||||
const bitsPerSample = 16;
|
|
||||||
const headerSize = 44;
|
|
||||||
const wav = Buffer.alloc(headerSize + pcm16.length);
|
|
||||||
const byteRate = (sampleRate * channels * bitsPerSample) / 8;
|
|
||||||
const blockAlign = (channels * bitsPerSample) / 8;
|
|
||||||
|
|
||||||
wav.write("RIFF", 0);
|
|
||||||
wav.writeUInt32LE(36 + pcm16.length, 4);
|
|
||||||
wav.write("WAVE", 8);
|
|
||||||
wav.write("fmt ", 12);
|
|
||||||
wav.writeUInt32LE(16, 16);
|
|
||||||
wav.writeUInt16LE(1, 20);
|
|
||||||
wav.writeUInt16LE(channels, 22);
|
|
||||||
wav.writeUInt32LE(sampleRate, 24);
|
|
||||||
wav.writeUInt32LE(byteRate, 28);
|
|
||||||
wav.writeUInt16LE(blockAlign, 32);
|
|
||||||
wav.writeUInt16LE(bitsPerSample, 34);
|
|
||||||
wav.write("data", 36);
|
|
||||||
wav.writeUInt32LE(pcm16.length, 40);
|
|
||||||
pcm16.copy(wav, 44);
|
|
||||||
|
|
||||||
return wav;
|
|
||||||
}
|
|
||||||
|
|
||||||
function sanitizeFilePart(input: string): string {
|
|
||||||
return input.replace(/[^a-zA-Z0-9._-]/g, "-");
|
|
||||||
}
|
|
||||||
|
|
||||||
function toFloatSamples(audio: unknown): Float32Array {
|
|
||||||
const raw = (audio as { samples?: unknown })?.samples;
|
|
||||||
if (raw instanceof Float32Array) {
|
|
||||||
return raw;
|
|
||||||
}
|
|
||||||
if (Array.isArray(raw)) {
|
|
||||||
return Float32Array.from(raw as number[]);
|
|
||||||
}
|
|
||||||
throw new Error("Unexpected Sherpa TTS output: missing Float32 samples");
|
|
||||||
}
|
|
||||||
|
|
||||||
function toSampleRate(audio: unknown, fallback: number): number {
|
|
||||||
const sampleRate = (audio as { sampleRate?: unknown })?.sampleRate;
|
|
||||||
if (typeof sampleRate === "number" && Number.isFinite(sampleRate) && sampleRate > 0) {
|
|
||||||
return sampleRate;
|
|
||||||
}
|
|
||||||
return fallback;
|
|
||||||
}
|
|
||||||
|
|
||||||
function createSherpaOfflineTts(params: {
|
|
||||||
modelId: "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16";
|
|
||||||
modelDir: string;
|
|
||||||
speed: number;
|
|
||||||
}): { tts: unknown; speed: number } {
|
|
||||||
const sherpa = loadSherpaOnnxNode();
|
|
||||||
const model =
|
|
||||||
params.modelId === "kokoro-en-v0_19"
|
|
||||||
? {
|
|
||||||
kokoro: {
|
|
||||||
model: `${params.modelDir}/model.onnx`,
|
|
||||||
voices: `${params.modelDir}/voices.bin`,
|
|
||||||
tokens: `${params.modelDir}/tokens.txt`,
|
|
||||||
dataDir: `${params.modelDir}/espeak-ng-data`,
|
|
||||||
lengthScale: 1.0,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
: {
|
|
||||||
kitten: {
|
|
||||||
model: `${params.modelDir}/model.fp16.onnx`,
|
|
||||||
voices: `${params.modelDir}/voices.bin`,
|
|
||||||
tokens: `${params.modelDir}/tokens.txt`,
|
|
||||||
dataDir: `${params.modelDir}/espeak-ng-data`,
|
|
||||||
lengthScale: 1.0,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
return {
|
|
||||||
tts: new sherpa.OfflineTts({
|
|
||||||
model,
|
|
||||||
numThreads: 2,
|
|
||||||
provider: "cpu",
|
|
||||||
maxNumSentences: 1,
|
|
||||||
}),
|
|
||||||
speed: params.speed,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function readStreamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
|
|
||||||
const chunks: Buffer[] = [];
|
|
||||||
for await (const chunk of stream as AsyncIterable<Buffer | Uint8Array>) {
|
|
||||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
||||||
}
|
|
||||||
return Buffer.concat(chunks);
|
|
||||||
}
|
|
||||||
|
|
||||||
function toCsv(rows: GeneratedSample[]): string {
|
|
||||||
const header = [
|
|
||||||
"model_id",
|
|
||||||
"voice_label",
|
|
||||||
"speaker_id",
|
|
||||||
"sample_rate",
|
|
||||||
"duration_seconds",
|
|
||||||
"relative_path",
|
|
||||||
];
|
|
||||||
|
|
||||||
const lines = [header.join(",")];
|
|
||||||
for (const row of rows) {
|
|
||||||
const cells = [
|
|
||||||
row.modelId,
|
|
||||||
row.voiceLabel,
|
|
||||||
row.speakerId === null ? "" : String(row.speakerId),
|
|
||||||
String(row.sampleRate),
|
|
||||||
row.durationSeconds.toFixed(3),
|
|
||||||
row.relativePath,
|
|
||||||
];
|
|
||||||
|
|
||||||
lines.push(
|
|
||||||
cells
|
|
||||||
.map((cell) => {
|
|
||||||
const escaped = cell.replace(/"/g, '""');
|
|
||||||
return escaped.includes(",") ? `"${escaped}"` : escaped;
|
|
||||||
})
|
|
||||||
.join(","),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return `${lines.join("\n")}\n`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const logger = createRootLogger({ level: "info", format: "pretty" });
|
|
||||||
const options = parseArgs(process.argv.slice(2));
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
modelsDir: options.modelsDir,
|
|
||||||
outputDir: options.outputDir,
|
|
||||||
modelIds: options.modelIds,
|
|
||||||
speed: options.speed,
|
|
||||||
text: options.text,
|
|
||||||
},
|
|
||||||
"Generating TTS matrix across all model/voice combinations",
|
|
||||||
);
|
|
||||||
|
|
||||||
await ensureLocalSpeechModels({
|
|
||||||
modelsDir: options.modelsDir,
|
|
||||||
modelIds: options.modelIds,
|
|
||||||
logger,
|
|
||||||
});
|
|
||||||
|
|
||||||
await mkdir(options.outputDir, { recursive: true });
|
|
||||||
|
|
||||||
const generated: GeneratedSample[] = [];
|
|
||||||
|
|
||||||
for (const modelId of options.modelIds) {
|
|
||||||
const modelDir = getLocalSpeechModelDir(options.modelsDir, modelId);
|
|
||||||
const modelOutputDir = path.join(options.outputDir, sanitizeFilePart(modelId));
|
|
||||||
await mkdir(modelOutputDir, { recursive: true });
|
|
||||||
|
|
||||||
if (modelId === "pocket-tts-onnx-int8") {
|
|
||||||
const tts = await PocketTtsOnnxTTS.create(
|
|
||||||
{
|
|
||||||
modelDir,
|
|
||||||
precision: "int8",
|
|
||||||
targetChunkMs: 50,
|
|
||||||
},
|
|
||||||
logger,
|
|
||||||
);
|
|
||||||
const result = await tts.synthesizeSpeech(options.text);
|
|
||||||
const pcm16 = await readStreamToBuffer(result.stream);
|
|
||||||
const sampleRate = parsePcmRateFromFormat(result.format, 24000) ?? 24000;
|
|
||||||
|
|
||||||
const voiceLabel = "default";
|
|
||||||
const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`;
|
|
||||||
const absPath = path.join(modelOutputDir, filename);
|
|
||||||
const relPath = path.relative(options.outputDir, absPath);
|
|
||||||
|
|
||||||
await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate));
|
|
||||||
|
|
||||||
generated.push({
|
|
||||||
modelId,
|
|
||||||
voiceLabel,
|
|
||||||
speakerId: null,
|
|
||||||
sampleRate,
|
|
||||||
durationSeconds: pcm16.length / 2 / sampleRate,
|
|
||||||
relativePath: relPath,
|
|
||||||
});
|
|
||||||
|
|
||||||
logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (modelId !== "kokoro-en-v0_19" && modelId !== "kitten-nano-en-v0_1-fp16") {
|
|
||||||
throw new Error(`Unsupported local Sherpa TTS model for this script: ${modelId}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const { tts, speed } = createSherpaOfflineTts({
|
|
||||||
modelId,
|
|
||||||
modelDir,
|
|
||||||
speed: options.speed,
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
const rawNumSpeakers = (tts as { numSpeakers?: unknown }).numSpeakers;
|
|
||||||
const numSpeakers =
|
|
||||||
typeof rawNumSpeakers === "number" && Number.isFinite(rawNumSpeakers) && rawNumSpeakers > 0
|
|
||||||
? Math.floor(rawNumSpeakers)
|
|
||||||
: 1;
|
|
||||||
const pad = Math.max(2, String(Math.max(0, numSpeakers - 1)).length);
|
|
||||||
const fallbackSampleRate =
|
|
||||||
typeof (tts as { sampleRate?: unknown }).sampleRate === "number"
|
|
||||||
? Math.max(1, Math.floor((tts as { sampleRate: number }).sampleRate))
|
|
||||||
: 24000;
|
|
||||||
|
|
||||||
logger.info({ modelId, numSpeakers }, "Discovered Sherpa speaker count");
|
|
||||||
|
|
||||||
for (let sid = 0; sid < numSpeakers; sid += 1) {
|
|
||||||
const audio = (
|
|
||||||
tts as {
|
|
||||||
generate: (config: { text: string; sid: number; speed: number }) => unknown;
|
|
||||||
}
|
|
||||||
).generate({
|
|
||||||
text: options.text,
|
|
||||||
sid,
|
|
||||||
speed,
|
|
||||||
});
|
|
||||||
const samples = toFloatSamples(audio);
|
|
||||||
const sampleRate = toSampleRate(audio, fallbackSampleRate);
|
|
||||||
const pcm16 = float32ToPcm16le(samples);
|
|
||||||
|
|
||||||
const voiceLabel = `sid-${String(sid).padStart(pad, "0")}`;
|
|
||||||
const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`;
|
|
||||||
const absPath = path.join(modelOutputDir, filename);
|
|
||||||
const relPath = path.relative(options.outputDir, absPath);
|
|
||||||
|
|
||||||
await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate));
|
|
||||||
|
|
||||||
generated.push({
|
|
||||||
modelId,
|
|
||||||
voiceLabel,
|
|
||||||
speakerId: sid,
|
|
||||||
sampleRate,
|
|
||||||
durationSeconds: pcm16.length / 2 / sampleRate,
|
|
||||||
relativePath: relPath,
|
|
||||||
});
|
|
||||||
|
|
||||||
logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample");
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
(tts as { free?: () => void }).free?.();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const manifest = {
|
|
||||||
createdAt: new Date().toISOString(),
|
|
||||||
text: options.text,
|
|
||||||
speed: options.speed,
|
|
||||||
modelsDir: options.modelsDir,
|
|
||||||
outputDir: options.outputDir,
|
|
||||||
namingConvention: "<model-id>/<model-id>__voice-<voice-label>.wav",
|
|
||||||
notes: [
|
|
||||||
"For Sherpa kitten/kokoro models, voice-label uses sid-<speaker-id>.",
|
|
||||||
"For Pocket TTS, voice-label is default.",
|
|
||||||
],
|
|
||||||
generated,
|
|
||||||
};
|
|
||||||
|
|
||||||
await writeFile(
|
|
||||||
path.join(options.outputDir, "manifest.json"),
|
|
||||||
`${JSON.stringify(manifest, null, 2)}\n`,
|
|
||||||
);
|
|
||||||
await writeFile(path.join(options.outputDir, "samples.csv"), toCsv(generated));
|
|
||||||
await writeFile(
|
|
||||||
path.join(options.outputDir, "README.txt"),
|
|
||||||
[
|
|
||||||
"Sherpa TTS sample matrix",
|
|
||||||
"",
|
|
||||||
`Sample text: ${options.text}`,
|
|
||||||
`Speed: ${options.speed}`,
|
|
||||||
"",
|
|
||||||
"Naming convention:",
|
|
||||||
" <model-id>/<model-id>__voice-<voice-label>.wav",
|
|
||||||
" voice-label = sid-<speaker-id> for Sherpa models",
|
|
||||||
" voice-label = default for Pocket TTS",
|
|
||||||
"",
|
|
||||||
`Total files: ${generated.length}`,
|
|
||||||
].join("\n"),
|
|
||||||
);
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
outputDir: options.outputDir,
|
|
||||||
generatedCount: generated.length,
|
|
||||||
models: Array.from(new Set(generated.map((item) => item.modelId))),
|
|
||||||
},
|
|
||||||
"TTS matrix generation complete",
|
|
||||||
);
|
|
||||||
@@ -29,7 +29,7 @@ function usage(): string {
|
|||||||
" npm run speech:transcribe:local -- ./sample.wav --out ./tmp/sample.transcript.txt",
|
" npm run speech:transcribe:local -- ./sample.wav --out ./tmp/sample.transcript.txt",
|
||||||
"",
|
"",
|
||||||
"Env fallbacks:",
|
"Env fallbacks:",
|
||||||
" PASEO_LOCAL_MODELS_DIR, PASEO_LOCAL_STT_MODEL",
|
" PASEO_LOCAL_MODELS_DIR",
|
||||||
].join("\n");
|
].join("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -49,9 +49,7 @@ function parseArgs(argv: string[]): CliOptions {
|
|||||||
|
|
||||||
const positional: string[] = [];
|
const positional: string[] = [];
|
||||||
let outPath: string | undefined;
|
let outPath: string | undefined;
|
||||||
let model = LocalSttModelIdSchema.parse(
|
let model = LocalSttModelIdSchema.parse(DEFAULT_LOCAL_STT_MODEL);
|
||||||
process.env.PASEO_LOCAL_STT_MODEL ?? DEFAULT_LOCAL_STT_MODEL,
|
|
||||||
);
|
|
||||||
let modelsDir = defaultModelsDir;
|
let modelsDir = defaultModelsDir;
|
||||||
|
|
||||||
for (let i = 0; i < argv.length; i++) {
|
for (let i = 0; i < argv.length; i++) {
|
||||||
|
|||||||
@@ -147,8 +147,8 @@ describe("paseo daemon bootstrap", () => {
|
|||||||
local: {
|
local: {
|
||||||
modelsDir: path.join(os.tmpdir(), `paseo-missing-models-${Date.now()}`),
|
modelsDir: path.join(os.tmpdir(), `paseo-missing-models-${Date.now()}`),
|
||||||
models: {
|
models: {
|
||||||
dictationStt: "parakeet-tdt-0.6b-v3-int8",
|
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||||
voiceStt: "parakeet-tdt-0.6b-v3-int8",
|
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||||
voiceTts: "kokoro-en-v0_19",
|
voiceTts: "kokoro-en-v0_19",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -39,35 +39,25 @@ async function readFixture(fileName: string): Promise<Buffer> {
|
|||||||
return readFile(fixturePath(fileName));
|
return readFile(fixturePath(fileName));
|
||||||
}
|
}
|
||||||
|
|
||||||
function hasSherpaZipformerModels(modelsDir: string): boolean {
|
function hasSherpaParakeetModels(modelsDir: string): boolean {
|
||||||
return (
|
return (
|
||||||
existsSync(
|
existsSync(
|
||||||
path.join(
|
path.join(modelsDir, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", "encoder.int8.onnx"),
|
||||||
modelsDir,
|
|
||||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
|
||||||
"encoder-epoch-99-avg-1.onnx",
|
|
||||||
),
|
|
||||||
) &&
|
) &&
|
||||||
existsSync(
|
existsSync(path.join(modelsDir, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", "tokens.txt"))
|
||||||
path.join(
|
|
||||||
modelsDir,
|
|
||||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
|
||||||
"tokens.txt",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function hasSherpaKittenModels(modelsDir: string): boolean {
|
function hasSherpaKokoroModels(modelsDir: string): boolean {
|
||||||
return (
|
return (
|
||||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "model.fp16.onnx")) &&
|
existsSync(path.join(modelsDir, "kokoro-en-v0_19", "model.onnx")) &&
|
||||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "voices.bin")) &&
|
existsSync(path.join(modelsDir, "kokoro-en-v0_19", "voices.bin")) &&
|
||||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "tokens.txt"))
|
existsSync(path.join(modelsDir, "kokoro-en-v0_19", "tokens.txt"))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const hasLocalSpeech =
|
const hasLocalSpeech =
|
||||||
hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir);
|
hasSherpaParakeetModels(localModelsDir) && hasSherpaKokoroModels(localModelsDir);
|
||||||
const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey);
|
const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey);
|
||||||
const speechTest = hasAnySpeech ? test : test.skip;
|
const speechTest = hasAnySpeech ? test : test.skip;
|
||||||
|
|
||||||
@@ -492,11 +482,10 @@ function resolveSpeechConfig() {
|
|||||||
local: {
|
local: {
|
||||||
modelsDir: localModelsDir,
|
modelsDir: localModelsDir,
|
||||||
models: {
|
models: {
|
||||||
dictationStt:
|
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||||
process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? "zipformer-bilingual-zh-en-2023-02-20",
|
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||||
voiceStt:
|
voiceTts: "kokoro-en-v0_19",
|
||||||
process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? "zipformer-bilingual-zh-en-2023-02-20",
|
voiceTtsSpeakerId: 0,
|
||||||
voiceTts: process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
-15
@@ -1,15 +0,0 @@
|
|||||||
// Type augmentation for onnxruntime-node to include runtime properties
|
|
||||||
// not exposed in the official type definitions
|
|
||||||
|
|
||||||
import type * as ort from "onnxruntime-node";
|
|
||||||
|
|
||||||
declare module "onnxruntime-node" {
|
|
||||||
interface InferenceSession {
|
|
||||||
/** Input tensor names (available at runtime but not in types) */
|
|
||||||
readonly inputNames: string[];
|
|
||||||
/** Output tensor names (available at runtime but not in types) */
|
|
||||||
readonly outputNames?: string[];
|
|
||||||
/** Input metadata for shape/type info (available at runtime but not in types) */
|
|
||||||
readonly inputMetadata?: unknown;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,629 +0,0 @@
|
|||||||
import { existsSync } from "node:fs";
|
|
||||||
import { readFile } from "node:fs/promises";
|
|
||||||
import { Readable } from "node:stream";
|
|
||||||
import type pino from "pino";
|
|
||||||
|
|
||||||
import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js";
|
|
||||||
import {
|
|
||||||
chunkBuffer,
|
|
||||||
float32ToPcm16le,
|
|
||||||
parsePcm16MonoWav,
|
|
||||||
pcm16leToFloat32,
|
|
||||||
} from "../../../audio.js";
|
|
||||||
import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
|
|
||||||
|
|
||||||
type OrtModule = typeof import("onnxruntime-node");
|
|
||||||
type OrtSession = import("onnxruntime-node").InferenceSession;
|
|
||||||
type OrtTensor = import("onnxruntime-node").Tensor;
|
|
||||||
|
|
||||||
function createSessionFeed(feed: Record<string, OrtTensor>): Record<string, OrtTensor> {
|
|
||||||
return feed;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SentencePieceProcessor {
|
|
||||||
encodeIds: (text: string) => number[];
|
|
||||||
load?: (modelPath: string) => unknown;
|
|
||||||
Load?: (modelPath: string) => unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
function assertFileExists(filePath: string, label: string): void {
|
|
||||||
if (!existsSync(filePath)) {
|
|
||||||
throw new Error(`Missing ${label}: ${filePath}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function product(dims: number[]): number {
|
|
||||||
let out = 1;
|
|
||||||
for (const d of dims) out *= d;
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeDims(dims: Array<number | string | null | undefined>): number[] {
|
|
||||||
// ONNX metadata can contain dynamic dimensions as strings (e.g. "batch") or -1.
|
|
||||||
// For state tensors we want a valid minimal shape, so coerce unknown/invalid dims to 1.
|
|
||||||
// Preserve explicit 0 dims (some models use empty initial state buffers with shape [0]).
|
|
||||||
return dims.map((d) => {
|
|
||||||
if (typeof d === "number" && Number.isFinite(d)) {
|
|
||||||
if (d === 0) return 0;
|
|
||||||
if (d > 0) return d;
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return 1;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function getSessionInputMeta(
|
|
||||||
session: OrtSession,
|
|
||||||
inputName: string,
|
|
||||||
): { type?: string; dims?: Array<number | string | null> } | undefined {
|
|
||||||
const metaAny = session.inputMetadata;
|
|
||||||
if (!Array.isArray(metaAny)) {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
const entry = metaAny.find((m) => m.name === inputName);
|
|
||||||
if (!entry) return undefined;
|
|
||||||
return { type: String(entry.type), dims: entry.shape };
|
|
||||||
}
|
|
||||||
|
|
||||||
function toBigInt64(values: number[]): BigInt64Array {
|
|
||||||
const out = new BigInt64Array(values.length);
|
|
||||||
for (let i = 0; i < values.length; i += 1) {
|
|
||||||
out[i] = BigInt(values[i]);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function randn(): number {
|
|
||||||
// Box–Muller
|
|
||||||
let u = 0;
|
|
||||||
let v = 0;
|
|
||||||
while (u === 0) u = Math.random();
|
|
||||||
while (v === 0) v = Math.random();
|
|
||||||
return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeTextForPocket(text: string): string {
|
|
||||||
const trimmed = text.trim();
|
|
||||||
if (!trimmed) {
|
|
||||||
throw new Error("Cannot synthesize empty text");
|
|
||||||
}
|
|
||||||
let out = trimmed;
|
|
||||||
if (out.length > 0 && /[A-Za-z0-9]$/.test(out)) {
|
|
||||||
out = `${out}.`;
|
|
||||||
}
|
|
||||||
if (out.length > 0 && /[a-z]/.test(out[0])) {
|
|
||||||
out = out[0].toUpperCase() + out.slice(1);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadOrt(): Promise<OrtModule> {
|
|
||||||
return import("onnxruntime-node");
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SentencePieceModule {
|
|
||||||
SentencePieceProcessor?: new () => SentencePieceProcessor;
|
|
||||||
default?:
|
|
||||||
| (new () => SentencePieceProcessor)
|
|
||||||
| { SentencePieceProcessor?: new () => SentencePieceProcessor };
|
|
||||||
}
|
|
||||||
|
|
||||||
function isSentencePieceModule(mod: unknown): mod is SentencePieceModule {
|
|
||||||
return mod !== null && typeof mod === "object";
|
|
||||||
}
|
|
||||||
|
|
||||||
function getSentencePieceProcessor(
|
|
||||||
mod: SentencePieceModule,
|
|
||||||
): (new () => SentencePieceProcessor) | undefined {
|
|
||||||
if (mod.SentencePieceProcessor) {
|
|
||||||
return mod.SentencePieceProcessor;
|
|
||||||
}
|
|
||||||
const defaultValue = mod.default;
|
|
||||||
if (
|
|
||||||
defaultValue &&
|
|
||||||
typeof defaultValue === "object" &&
|
|
||||||
"SentencePieceProcessor" in defaultValue
|
|
||||||
) {
|
|
||||||
return defaultValue.SentencePieceProcessor;
|
|
||||||
}
|
|
||||||
if (typeof defaultValue === "function") {
|
|
||||||
return defaultValue as new () => SentencePieceProcessor;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadSentencePiece(tokenizerModelPath: string): Promise<SentencePieceProcessor> {
|
|
||||||
const mod = await import("@sctg/sentencepiece-js");
|
|
||||||
|
|
||||||
if (!isSentencePieceModule(mod)) {
|
|
||||||
throw new Error("@sctg/sentencepiece-js module has unexpected shape");
|
|
||||||
}
|
|
||||||
|
|
||||||
const Processor = getSentencePieceProcessor(mod);
|
|
||||||
|
|
||||||
if (!Processor) {
|
|
||||||
throw new Error("Failed to load SentencePiece processor from @sctg/sentencepiece-js");
|
|
||||||
}
|
|
||||||
|
|
||||||
const sp = new Processor();
|
|
||||||
|
|
||||||
if (typeof sp.load === "function") {
|
|
||||||
await sp.load(tokenizerModelPath);
|
|
||||||
} else if (typeof sp.Load === "function") {
|
|
||||||
sp.Load(tokenizerModelPath);
|
|
||||||
} else {
|
|
||||||
throw new Error("SentencePiece processor does not expose load()/Load()");
|
|
||||||
}
|
|
||||||
|
|
||||||
return sp;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getOrtProviders(ort: OrtModule, device: "auto" | "cpu" | "cuda"): string[] {
|
|
||||||
// NOTE: onnxruntime-node uses backend names like "cpu"/"coreml"/"webgpu" (not "CPUExecutionProvider").
|
|
||||||
if (device === "cpu") return ["cpu"];
|
|
||||||
if (device === "cuda") return ["cuda", "cpu"];
|
|
||||||
// auto
|
|
||||||
// CoreML EP does not support some dynamic/zero-length shapes used by Pocket TTS (e.g. [1, 0, 32]).
|
|
||||||
// Default to CPU to keep behavior predictable across platforms.
|
|
||||||
void ort;
|
|
||||||
return ["cpu"];
|
|
||||||
}
|
|
||||||
|
|
||||||
function createZeroTensorForInput(
|
|
||||||
ort: OrtModule,
|
|
||||||
session: OrtSession,
|
|
||||||
inputName: string,
|
|
||||||
): OrtTensor {
|
|
||||||
const meta = getSessionInputMeta(session, inputName);
|
|
||||||
const dims = normalizeDims(meta?.dims ?? []);
|
|
||||||
if (dims.length === 0) {
|
|
||||||
throw new Error(`Missing input metadata shape for ${inputName}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const type = (meta?.type ?? "float32").toLowerCase();
|
|
||||||
const size = product(dims);
|
|
||||||
|
|
||||||
if (type.includes("int64")) {
|
|
||||||
return new ort.Tensor("int64", new BigInt64Array(size), dims);
|
|
||||||
}
|
|
||||||
if (type.includes("bool")) {
|
|
||||||
return new ort.Tensor("bool", new Uint8Array(size), dims);
|
|
||||||
}
|
|
||||||
return new ort.Tensor("float32", new Float32Array(size), dims);
|
|
||||||
}
|
|
||||||
|
|
||||||
function initState(session: OrtSession, ort: OrtModule): Record<string, OrtTensor> {
|
|
||||||
const out: Record<string, OrtTensor> = {};
|
|
||||||
for (const name of session.inputNames) {
|
|
||||||
if (name.startsWith("state_")) {
|
|
||||||
out[name] = createZeroTensorForInput(ort, session, name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function updateStateFromOutputs(
|
|
||||||
state: Record<string, OrtTensor>,
|
|
||||||
outputs: Record<string, OrtTensor>,
|
|
||||||
): void {
|
|
||||||
for (const [name, tensor] of Object.entries(outputs)) {
|
|
||||||
if (!name.startsWith("out_state_")) continue;
|
|
||||||
const idx = Number.parseInt(name.replace("out_state_", ""), 10);
|
|
||||||
if (Number.isFinite(idx)) {
|
|
||||||
state[`state_${idx}`] = tensor;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
interface OrtTensorWithData {
|
|
||||||
data: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
function tensorDataFloat32(t: OrtTensor): Float32Array {
|
|
||||||
const tensorWithData = t as OrtTensorWithData;
|
|
||||||
const data = tensorWithData.data;
|
|
||||||
if (data instanceof Float32Array) return data;
|
|
||||||
if (Array.isArray(data)) return Float32Array.from(data);
|
|
||||||
throw new Error("Unexpected tensor data type (expected Float32Array)");
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PocketTtsOnnxConfig {
|
|
||||||
modelDir: string;
|
|
||||||
precision?: "int8" | "fp32";
|
|
||||||
device?: "auto" | "cpu" | "cuda";
|
|
||||||
temperature?: number;
|
|
||||||
lsdSteps?: number;
|
|
||||||
maxFrames?: number;
|
|
||||||
framesAfterEos?: number;
|
|
||||||
firstChunkFrames?: number;
|
|
||||||
maxChunkFrames?: number;
|
|
||||||
targetChunkMs?: number;
|
|
||||||
referenceAudioFile?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
class PocketTtsOnnxEngine {
|
|
||||||
static readonly SAMPLE_RATE = 24000;
|
|
||||||
static readonly SAMPLES_PER_FRAME = 1920;
|
|
||||||
|
|
||||||
private readonly ort: OrtModule;
|
|
||||||
|
|
||||||
private readonly temperature: number;
|
|
||||||
private readonly lsdSteps: number;
|
|
||||||
private readonly maxFrames: number;
|
|
||||||
private readonly framesAfterEos: number;
|
|
||||||
|
|
||||||
private readonly firstChunkFrames: number;
|
|
||||||
private readonly maxChunkFrames: number;
|
|
||||||
|
|
||||||
private readonly tokenizer: SentencePieceProcessor;
|
|
||||||
private readonly textConditioner: OrtSession;
|
|
||||||
private readonly flowLmMain: OrtSession;
|
|
||||||
private readonly flowLmFlow: OrtSession;
|
|
||||||
private readonly mimiDecoder: OrtSession;
|
|
||||||
|
|
||||||
private readonly stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
|
|
||||||
private readonly voiceEmbeddings: OrtTensor;
|
|
||||||
|
|
||||||
private constructor(args: {
|
|
||||||
ort: OrtModule;
|
|
||||||
temperature: number;
|
|
||||||
lsdSteps: number;
|
|
||||||
maxFrames: number;
|
|
||||||
framesAfterEos: number;
|
|
||||||
firstChunkFrames: number;
|
|
||||||
maxChunkFrames: number;
|
|
||||||
tokenizer: SentencePieceProcessor;
|
|
||||||
textConditioner: OrtSession;
|
|
||||||
flowLmMain: OrtSession;
|
|
||||||
flowLmFlow: OrtSession;
|
|
||||||
mimiDecoder: OrtSession;
|
|
||||||
stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
|
|
||||||
voiceEmbeddings: OrtTensor;
|
|
||||||
}) {
|
|
||||||
this.ort = args.ort;
|
|
||||||
this.temperature = args.temperature;
|
|
||||||
this.lsdSteps = args.lsdSteps;
|
|
||||||
this.maxFrames = args.maxFrames;
|
|
||||||
this.framesAfterEos = args.framesAfterEos;
|
|
||||||
this.firstChunkFrames = args.firstChunkFrames;
|
|
||||||
this.maxChunkFrames = args.maxChunkFrames;
|
|
||||||
this.tokenizer = args.tokenizer;
|
|
||||||
this.textConditioner = args.textConditioner;
|
|
||||||
this.flowLmMain = args.flowLmMain;
|
|
||||||
this.flowLmFlow = args.flowLmFlow;
|
|
||||||
this.mimiDecoder = args.mimiDecoder;
|
|
||||||
this.stBuffers = args.stBuffers;
|
|
||||||
this.voiceEmbeddings = args.voiceEmbeddings;
|
|
||||||
}
|
|
||||||
|
|
||||||
static async create(
|
|
||||||
config: PocketTtsOnnxConfig,
|
|
||||||
logger: pino.Logger,
|
|
||||||
): Promise<PocketTtsOnnxEngine> {
|
|
||||||
const log = logger.child({
|
|
||||||
module: "speech",
|
|
||||||
provider: "pocket-tts",
|
|
||||||
component: "onnx-engine",
|
|
||||||
});
|
|
||||||
|
|
||||||
const modelDir = config.modelDir;
|
|
||||||
const onnxDir = `${modelDir}/onnx`;
|
|
||||||
const precision = config.precision ?? "int8";
|
|
||||||
const device = config.device ?? "auto";
|
|
||||||
const temperature = config.temperature ?? 0.7;
|
|
||||||
const lsdSteps = config.lsdSteps ?? 10;
|
|
||||||
const maxFrames = config.maxFrames ?? 500;
|
|
||||||
const framesAfterEos = config.framesAfterEos ?? 3;
|
|
||||||
const firstChunkFrames = config.firstChunkFrames ?? 2;
|
|
||||||
const maxChunkFrames = config.maxChunkFrames ?? 15;
|
|
||||||
|
|
||||||
const tokenizerPath = `${modelDir}/tokenizer.model`;
|
|
||||||
const referenceAudioFile = config.referenceAudioFile ?? `${modelDir}/reference_sample.wav`;
|
|
||||||
|
|
||||||
const flowMainFile = precision === "int8" ? "flow_lm_main_int8.onnx" : "flow_lm_main.onnx";
|
|
||||||
const flowFlowFile = precision === "int8" ? "flow_lm_flow_int8.onnx" : "flow_lm_flow.onnx";
|
|
||||||
const decoderFile = precision === "int8" ? "mimi_decoder_int8.onnx" : "mimi_decoder.onnx";
|
|
||||||
|
|
||||||
assertFileExists(`${onnxDir}/mimi_encoder.onnx`, "PocketTTS mimi_encoder");
|
|
||||||
assertFileExists(`${onnxDir}/text_conditioner.onnx`, "PocketTTS text_conditioner");
|
|
||||||
assertFileExists(`${onnxDir}/${flowMainFile}`, "PocketTTS flow_lm_main");
|
|
||||||
assertFileExists(`${onnxDir}/${flowFlowFile}`, "PocketTTS flow_lm_flow");
|
|
||||||
assertFileExists(`${onnxDir}/${decoderFile}`, "PocketTTS mimi_decoder");
|
|
||||||
assertFileExists(tokenizerPath, "PocketTTS tokenizer.model");
|
|
||||||
assertFileExists(referenceAudioFile, "PocketTTS reference_sample.wav");
|
|
||||||
|
|
||||||
const ort = await loadOrt();
|
|
||||||
const providers = getOrtProviders(ort, device);
|
|
||||||
|
|
||||||
const [tokenizer, mimiEncoder, textConditioner, flowLmMain, flowLmFlow, mimiDecoder] =
|
|
||||||
await Promise.all([
|
|
||||||
loadSentencePiece(tokenizerPath),
|
|
||||||
ort.InferenceSession.create(`${onnxDir}/mimi_encoder.onnx`, {
|
|
||||||
executionProviders: providers,
|
|
||||||
}),
|
|
||||||
ort.InferenceSession.create(`${onnxDir}/text_conditioner.onnx`, {
|
|
||||||
executionProviders: providers,
|
|
||||||
}),
|
|
||||||
ort.InferenceSession.create(`${onnxDir}/${flowMainFile}`, {
|
|
||||||
executionProviders: providers,
|
|
||||||
}),
|
|
||||||
ort.InferenceSession.create(`${onnxDir}/${flowFlowFile}`, {
|
|
||||||
executionProviders: providers,
|
|
||||||
}),
|
|
||||||
ort.InferenceSession.create(`${onnxDir}/${decoderFile}`, { executionProviders: providers }),
|
|
||||||
]);
|
|
||||||
|
|
||||||
// Precompute flow matching time-step buffers.
|
|
||||||
const stBuffers: Array<{ s: OrtTensor; t: OrtTensor }> = [];
|
|
||||||
for (let j = 0; j < lsdSteps; j += 1) {
|
|
||||||
const s = j / lsdSteps;
|
|
||||||
const t = s + 1.0 / lsdSteps;
|
|
||||||
stBuffers.push({
|
|
||||||
s: new ort.Tensor("float32", new Float32Array([s]), [1, 1]),
|
|
||||||
t: new ort.Tensor("float32", new Float32Array([t]), [1, 1]),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Precompute reference voice embeddings once.
|
|
||||||
const refWav = await readFile(referenceAudioFile);
|
|
||||||
const parsed = parsePcm16MonoWav(refWav);
|
|
||||||
let pcm16 = parsed.pcm16;
|
|
||||||
if (parsed.sampleRate !== PocketTtsOnnxEngine.SAMPLE_RATE) {
|
|
||||||
const resampler = new Pcm16MonoResampler({
|
|
||||||
inputRate: parsed.sampleRate,
|
|
||||||
outputRate: PocketTtsOnnxEngine.SAMPLE_RATE,
|
|
||||||
});
|
|
||||||
pcm16 = resampler.processChunk(pcm16);
|
|
||||||
}
|
|
||||||
const floatAudio = pcm16leToFloat32(pcm16);
|
|
||||||
const audioTensor = new ort.Tensor("float32", floatAudio, [1, 1, floatAudio.length]);
|
|
||||||
|
|
||||||
const encoded = await mimiEncoder.run({ audio: audioTensor });
|
|
||||||
const firstOutName = mimiEncoder.outputNames?.[0];
|
|
||||||
const encodedRecord = encoded as Record<string, OrtTensor>;
|
|
||||||
const voiceEmb = firstOutName ? encodedRecord[firstOutName] : Object.values(encodedRecord)[0];
|
|
||||||
if (!voiceEmb) {
|
|
||||||
throw new Error("PocketTTS mimi_encoder: missing output");
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info({ precision, device, providers, lsdSteps, temperature }, "PocketTTS ONNX initialized");
|
|
||||||
|
|
||||||
return new PocketTtsOnnxEngine({
|
|
||||||
ort,
|
|
||||||
temperature,
|
|
||||||
lsdSteps,
|
|
||||||
maxFrames,
|
|
||||||
framesAfterEos,
|
|
||||||
firstChunkFrames,
|
|
||||||
maxChunkFrames,
|
|
||||||
tokenizer,
|
|
||||||
textConditioner,
|
|
||||||
flowLmMain,
|
|
||||||
flowLmFlow,
|
|
||||||
mimiDecoder,
|
|
||||||
stBuffers,
|
|
||||||
voiceEmbeddings: voiceEmb,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private tokenize(text: string): OrtTensor {
|
|
||||||
const normalized = normalizeTextForPocket(text);
|
|
||||||
const ids = this.tokenizer.encodeIds(normalized);
|
|
||||||
const data = toBigInt64(ids ?? []);
|
|
||||||
return new this.ort.Tensor("int64", data, [1, data.length]);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async runTextConditioner(tokenIds: OrtTensor): Promise<OrtTensor> {
|
|
||||||
const feed: Record<string, OrtTensor> = { token_ids: tokenIds };
|
|
||||||
const out = await this.textConditioner.run(feed);
|
|
||||||
const firstOutName = this.textConditioner.outputNames?.[0];
|
|
||||||
const outRecord = out as Record<string, OrtTensor>;
|
|
||||||
const t = firstOutName ? outRecord[firstOutName] : Object.values(outRecord)[0];
|
|
||||||
if (!t) throw new Error("PocketTTS text_conditioner: missing output");
|
|
||||||
return t;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async *runFlowLm(textEmbeddings: OrtTensor): AsyncGenerator<Float32Array> {
|
|
||||||
const ort = this.ort;
|
|
||||||
const state = initState(this.flowLmMain, ort);
|
|
||||||
|
|
||||||
const emptySeq = new ort.Tensor("float32", new Float32Array(0), [1, 0, 32]);
|
|
||||||
const emptyText = new ort.Tensor("float32", new Float32Array(0), [1, 0, 1024]);
|
|
||||||
|
|
||||||
// Voice conditioning pass
|
|
||||||
const resVoice = await this.flowLmMain.run(
|
|
||||||
createSessionFeed({
|
|
||||||
sequence: emptySeq,
|
|
||||||
text_embeddings: this.voiceEmbeddings,
|
|
||||||
...state,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
updateStateFromOutputs(state, resVoice as Record<string, OrtTensor>);
|
|
||||||
|
|
||||||
// Text conditioning pass
|
|
||||||
const resText = await this.flowLmMain.run(
|
|
||||||
createSessionFeed({
|
|
||||||
sequence: emptySeq,
|
|
||||||
text_embeddings: textEmbeddings,
|
|
||||||
...state,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
updateStateFromOutputs(state, resText as Record<string, OrtTensor>);
|
|
||||||
|
|
||||||
// Autoregressive generation
|
|
||||||
const curr = new Float32Array(32);
|
|
||||||
curr.fill(Number.NaN);
|
|
||||||
let currTensor = new ort.Tensor("float32", curr, [1, 1, 32]);
|
|
||||||
|
|
||||||
const dt = 1.0 / this.lsdSteps;
|
|
||||||
let eosStep: number | null = null;
|
|
||||||
|
|
||||||
for (let step = 0; step < this.maxFrames; step += 1) {
|
|
||||||
const resStep = await this.flowLmMain.run(
|
|
||||||
createSessionFeed({
|
|
||||||
sequence: currTensor,
|
|
||||||
text_embeddings: emptyText,
|
|
||||||
...state,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
const outputNames = this.flowLmMain.outputNames;
|
|
||||||
const resStepRecord = resStep as Record<string, OrtTensor>;
|
|
||||||
const conditioningName = outputNames?.[0] ?? Object.keys(resStepRecord)[0];
|
|
||||||
const eosName = outputNames?.[1] ?? Object.keys(resStepRecord)[1];
|
|
||||||
|
|
||||||
const conditioning = resStepRecord[conditioningName];
|
|
||||||
const eos = resStepRecord[eosName];
|
|
||||||
if (!conditioning || !eos) {
|
|
||||||
throw new Error("PocketTTS flow_lm_main: missing conditioning/EOS outputs");
|
|
||||||
}
|
|
||||||
updateStateFromOutputs(state, resStepRecord);
|
|
||||||
|
|
||||||
const eosData = tensorDataFloat32(eos);
|
|
||||||
if (eosData[0] > -4.0 && eosStep === null) {
|
|
||||||
eosStep = step;
|
|
||||||
}
|
|
||||||
if (eosStep !== null && step >= eosStep + this.framesAfterEos) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Flow matching with external Euler loop.
|
|
||||||
const std = this.temperature > 0 ? Math.sqrt(this.temperature) : 0;
|
|
||||||
const x = new Float32Array(32);
|
|
||||||
if (std > 0) {
|
|
||||||
for (let i = 0; i < x.length; i += 1) {
|
|
||||||
x[i] = randn() * std;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const st of this.stBuffers) {
|
|
||||||
const xTensor = new ort.Tensor("float32", x, [1, 32]);
|
|
||||||
const flowOut = await this.flowLmFlow.run(
|
|
||||||
createSessionFeed({
|
|
||||||
c: conditioning,
|
|
||||||
s: st.s,
|
|
||||||
t: st.t,
|
|
||||||
x: xTensor,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
const first = this.flowLmFlow.outputNames?.[0];
|
|
||||||
const flowOutRecord = flowOut as Record<string, OrtTensor>;
|
|
||||||
const flowTensor = first ? flowOutRecord[first] : Object.values(flowOutRecord)[0];
|
|
||||||
if (!flowTensor) throw new Error("PocketTTS flow_lm_flow: missing output");
|
|
||||||
const delta = tensorDataFloat32(flowTensor);
|
|
||||||
for (let i = 0; i < x.length; i += 1) {
|
|
||||||
x[i] = x[i] + delta[i] * dt;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
yield x;
|
|
||||||
currTensor = new ort.Tensor("float32", x, [1, 1, 32]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async decodeLatentsChunk(
|
|
||||||
frames: Float32Array[],
|
|
||||||
state: Record<string, OrtTensor>,
|
|
||||||
): Promise<Float32Array> {
|
|
||||||
const ort = this.ort;
|
|
||||||
const frameCount = frames.length;
|
|
||||||
const flattened = new Float32Array(frameCount * 32);
|
|
||||||
for (let i = 0; i < frameCount; i += 1) {
|
|
||||||
flattened.set(frames[i], i * 32);
|
|
||||||
}
|
|
||||||
const latent = new ort.Tensor("float32", flattened, [1, frameCount, 32]);
|
|
||||||
|
|
||||||
const out = await this.mimiDecoder.run(createSessionFeed({ latent, ...state }));
|
|
||||||
const outRecord = out as Record<string, OrtTensor>;
|
|
||||||
updateStateFromOutputs(state, outRecord);
|
|
||||||
|
|
||||||
const firstOutName = this.mimiDecoder.outputNames?.[0];
|
|
||||||
const audioTensor = firstOutName ? outRecord[firstOutName] : Object.values(outRecord)[0];
|
|
||||||
if (!audioTensor) {
|
|
||||||
throw new Error("PocketTTS mimi_decoder: missing audio output");
|
|
||||||
}
|
|
||||||
return tensorDataFloat32(audioTensor);
|
|
||||||
}
|
|
||||||
|
|
||||||
async *streamAudio(text: string): AsyncGenerator<Float32Array> {
|
|
||||||
const tokenIds = this.tokenize(text);
|
|
||||||
const textEmb = await this.runTextConditioner(tokenIds);
|
|
||||||
|
|
||||||
const decoderState = initState(this.mimiDecoder, this.ort);
|
|
||||||
|
|
||||||
const generated: Float32Array[] = [];
|
|
||||||
let decodedFrames = 0;
|
|
||||||
|
|
||||||
for await (const latent of this.runFlowLm(textEmb)) {
|
|
||||||
generated.push(latent);
|
|
||||||
const pending = generated.length - decodedFrames;
|
|
||||||
|
|
||||||
let chunkSize = 0;
|
|
||||||
if (decodedFrames === 0) {
|
|
||||||
if (pending >= this.firstChunkFrames) {
|
|
||||||
chunkSize = this.firstChunkFrames;
|
|
||||||
}
|
|
||||||
} else if (pending >= this.maxChunkFrames) {
|
|
||||||
chunkSize = this.maxChunkFrames;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (chunkSize > 0) {
|
|
||||||
const audio = await this.decodeLatentsChunk(
|
|
||||||
generated.slice(decodedFrames, decodedFrames + chunkSize),
|
|
||||||
decoderState,
|
|
||||||
);
|
|
||||||
decodedFrames += chunkSize;
|
|
||||||
yield audio;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (decodedFrames < generated.length) {
|
|
||||||
const audio = await this.decodeLatentsChunk(generated.slice(decodedFrames), decoderState);
|
|
||||||
yield audio;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class PocketTtsOnnxTTS implements TextToSpeechProvider {
|
|
||||||
private readonly engine: PocketTtsOnnxEngine;
|
|
||||||
private readonly chunkMs: number;
|
|
||||||
private readonly logger: pino.Logger;
|
|
||||||
|
|
||||||
private constructor(engine: PocketTtsOnnxEngine, logger: pino.Logger, chunkMs: number) {
|
|
||||||
this.engine = engine;
|
|
||||||
this.chunkMs = chunkMs;
|
|
||||||
this.logger = logger.child({ module: "speech", provider: "pocket-tts", component: "tts" });
|
|
||||||
}
|
|
||||||
|
|
||||||
static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise<PocketTtsOnnxTTS> {
|
|
||||||
const engine = await PocketTtsOnnxEngine.create(config, logger);
|
|
||||||
const chunkMs = config.targetChunkMs ?? 50;
|
|
||||||
return new PocketTtsOnnxTTS(engine, logger, chunkMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
async synthesizeSpeech(text: string): Promise<SpeechStreamResult> {
|
|
||||||
const start = Date.now();
|
|
||||||
const sampleRate = PocketTtsOnnxEngine.SAMPLE_RATE;
|
|
||||||
|
|
||||||
const pcmChunkBytes = Math.max(2, Math.round((sampleRate * this.chunkMs) / 1000) * 2);
|
|
||||||
|
|
||||||
const iterable = (async function* (engine: PocketTtsOnnxEngine) {
|
|
||||||
for await (const floatChunk of engine.streamAudio(text)) {
|
|
||||||
const pcm = float32ToPcm16le(floatChunk);
|
|
||||||
for (const chunk of chunkBuffer(pcm, pcmChunkBytes)) {
|
|
||||||
yield chunk;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})(this.engine);
|
|
||||||
|
|
||||||
this.logger.debug(
|
|
||||||
{ ms: Date.now() - start, textLength: text.length },
|
|
||||||
"PocketTTS stream ready",
|
|
||||||
);
|
|
||||||
|
|
||||||
return {
|
|
||||||
stream: Readable.from(iterable),
|
|
||||||
format: `pcm;rate=${sampleRate}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -4,7 +4,6 @@ import type { PaseoSpeechConfig } from "../../../bootstrap.js";
|
|||||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
|
import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
|
||||||
import type { RequestedSpeechProviders } from "../../speech-types.js";
|
import type { RequestedSpeechProviders } from "../../speech-types.js";
|
||||||
import type { TurnDetectionProvider } from "../../turn-detection-provider.js";
|
import type { TurnDetectionProvider } from "../../turn-detection-provider.js";
|
||||||
import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js";
|
|
||||||
import {
|
import {
|
||||||
getLocalSpeechModelDir,
|
getLocalSpeechModelDir,
|
||||||
DEFAULT_LOCAL_STT_MODEL,
|
DEFAULT_LOCAL_STT_MODEL,
|
||||||
@@ -16,20 +15,15 @@ import {
|
|||||||
type LocalTtsModelId,
|
type LocalTtsModelId,
|
||||||
} from "./models.js";
|
} from "./models.js";
|
||||||
import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
|
import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
|
||||||
import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js";
|
|
||||||
import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
|
import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
|
||||||
import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
|
import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
|
||||||
import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js";
|
|
||||||
import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js";
|
|
||||||
import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
|
import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
|
||||||
import {
|
import {
|
||||||
ensureSileroVadModel,
|
ensureSileroVadModel,
|
||||||
SherpaSileroTurnDetectionProvider,
|
SherpaSileroTurnDetectionProvider,
|
||||||
} from "./sherpa/silero-vad-provider.js";
|
} from "./sherpa/silero-vad-provider.js";
|
||||||
|
|
||||||
type LocalSttEngine =
|
type LocalSttEngine = SherpaOfflineRecognizerEngine;
|
||||||
| { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
|
|
||||||
| { kind: "online"; engine: SherpaOnlineRecognizerEngine };
|
|
||||||
|
|
||||||
interface ResolvedLocalModels {
|
interface ResolvedLocalModels {
|
||||||
dictationLocalSttModel: LocalSttModelId;
|
dictationLocalSttModel: LocalSttModelId;
|
||||||
@@ -117,70 +111,21 @@ async function createLocalSttEngine(params: {
|
|||||||
}): Promise<LocalSttEngine> {
|
}): Promise<LocalSttEngine> {
|
||||||
const { modelId, modelsDir, logger } = params;
|
const { modelId, modelsDir, logger } = params;
|
||||||
|
|
||||||
if (modelId === "parakeet-tdt-0.6b-v3-int8" || modelId === "parakeet-tdt-0.6b-v2-int8") {
|
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
||||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
return new SherpaOfflineRecognizerEngine(
|
||||||
return {
|
{
|
||||||
kind: "offline",
|
model: {
|
||||||
engine: new SherpaOfflineRecognizerEngine(
|
kind: "nemo_transducer",
|
||||||
{
|
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||||
model: {
|
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||||
kind: "nemo_transducer",
|
joiner: `${modelDir}/joiner.int8.onnx`,
|
||||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
tokens: `${modelDir}/tokens.txt`,
|
||||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
},
|
||||||
joiner: `${modelDir}/joiner.int8.onnx`,
|
numThreads: 2,
|
||||||
tokens: `${modelDir}/tokens.txt`,
|
debug: 0,
|
||||||
},
|
},
|
||||||
numThreads: 2,
|
logger,
|
||||||
debug: 0,
|
);
|
||||||
},
|
|
||||||
logger,
|
|
||||||
),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
if (modelId === "paraformer-bilingual-zh-en") {
|
|
||||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
|
||||||
return {
|
|
||||||
kind: "online",
|
|
||||||
engine: new SherpaOnlineRecognizerEngine(
|
|
||||||
{
|
|
||||||
model: {
|
|
||||||
kind: "paraformer",
|
|
||||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
|
||||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
|
||||||
tokens: `${modelDir}/tokens.txt`,
|
|
||||||
},
|
|
||||||
numThreads: 1,
|
|
||||||
debug: 0,
|
|
||||||
},
|
|
||||||
logger,
|
|
||||||
),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
|
|
||||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
|
||||||
return {
|
|
||||||
kind: "online",
|
|
||||||
engine: new SherpaOnlineRecognizerEngine(
|
|
||||||
{
|
|
||||||
model: {
|
|
||||||
kind: "transducer",
|
|
||||||
encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
|
|
||||||
decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
|
|
||||||
joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
|
|
||||||
tokens: `${modelDir}/tokens.txt`,
|
|
||||||
modelType: "zipformer",
|
|
||||||
},
|
|
||||||
numThreads: 1,
|
|
||||||
debug: 0,
|
|
||||||
},
|
|
||||||
logger,
|
|
||||||
),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(`Unsupported local STT model '${modelId}'`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type LocalConfig = NonNullable<PaseoSpeechConfig["local"]>;
|
type LocalConfig = NonNullable<PaseoSpeechConfig["local"]>;
|
||||||
@@ -219,13 +164,7 @@ async function initializeLocalVoiceStt(params: {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
const voiceEngine = await getLocalSttEngine(modelId);
|
const voiceEngine = await getLocalSttEngine(modelId);
|
||||||
if (voiceEngine?.kind === "offline") {
|
return voiceEngine ? new SherpaOnnxParakeetSTT({ engine: voiceEngine }, logger) : null;
|
||||||
return new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
|
|
||||||
}
|
|
||||||
if (voiceEngine?.kind === "online") {
|
|
||||||
return new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function initializeLocalDictationStt(params: {
|
async function initializeLocalDictationStt(params: {
|
||||||
@@ -243,18 +182,11 @@ async function initializeLocalDictationStt(params: {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
const dictationEngine = await getLocalSttEngine(modelId);
|
const dictationEngine = await getLocalSttEngine(modelId);
|
||||||
if (dictationEngine?.kind === "offline") {
|
if (dictationEngine) {
|
||||||
return {
|
return {
|
||||||
id: "local",
|
id: "local",
|
||||||
createSession: () =>
|
createSession: () =>
|
||||||
new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine }),
|
||||||
};
|
|
||||||
}
|
|
||||||
if (dictationEngine?.kind === "online") {
|
|
||||||
return {
|
|
||||||
id: "local",
|
|
||||||
createSession: () =>
|
|
||||||
new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
@@ -276,16 +208,6 @@ async function initializeLocalVoiceTts(params: {
|
|||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
|
const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
|
||||||
if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
|
|
||||||
return await PocketTtsOnnxTTS.create(
|
|
||||||
{
|
|
||||||
modelDir,
|
|
||||||
precision: "int8",
|
|
||||||
targetChunkMs: 50,
|
|
||||||
},
|
|
||||||
logger,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return new SherpaOnnxTTS(
|
return new SherpaOnnxTTS(
|
||||||
{
|
{
|
||||||
preset: localModels.voiceLocalTtsModel,
|
preset: localModels.voiceLocalTtsModel,
|
||||||
@@ -401,7 +323,7 @@ export async function initializeLocalSpeechServices(params: {
|
|||||||
maybeFreeable.free();
|
maybeFreeable.free();
|
||||||
}
|
}
|
||||||
for (const engine of localSttEngines.values()) {
|
for (const engine of localSttEngines.values()) {
|
||||||
engine.engine.free();
|
engine.free();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -1,44 +1,19 @@
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
|
||||||
export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts";
|
export type SherpaOnnxModelKind = "stt-offline" | "tts";
|
||||||
|
|
||||||
type DefaultModelRole = "stt" | "tts";
|
type DefaultModelRole = "stt" | "tts";
|
||||||
|
|
||||||
interface SherpaOnnxCatalogEntry {
|
interface SherpaOnnxCatalogEntry {
|
||||||
kind: SherpaOnnxModelKind;
|
kind: SherpaOnnxModelKind;
|
||||||
archiveUrl?: string;
|
archiveUrl: string;
|
||||||
downloadFiles?: Array<{ url: string; relPath: string }>;
|
|
||||||
extractedDir: string;
|
extractedDir: string;
|
||||||
requiredFiles: string[];
|
requiredFiles: string[];
|
||||||
description: string;
|
description: string;
|
||||||
aliases?: readonly string[];
|
|
||||||
defaultFor?: DefaultModelRole;
|
defaultFor?: DefaultModelRole;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const SHERPA_ONNX_MODEL_CATALOG = {
|
export const SHERPA_ONNX_MODEL_CATALOG = {
|
||||||
"zipformer-bilingual-zh-en-2023-02-20": {
|
|
||||||
kind: "stt-online",
|
|
||||||
archiveUrl:
|
|
||||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2",
|
|
||||||
extractedDir: "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
|
||||||
requiredFiles: [
|
|
||||||
"encoder-epoch-99-avg-1.onnx",
|
|
||||||
"decoder-epoch-99-avg-1.onnx",
|
|
||||||
"joiner-epoch-99-avg-1.onnx",
|
|
||||||
"tokens.txt",
|
|
||||||
],
|
|
||||||
description: "Streaming Zipformer transducer (fast, good accuracy).",
|
|
||||||
aliases: ["zipformer", "zipformer-bilingual"],
|
|
||||||
},
|
|
||||||
"paraformer-bilingual-zh-en": {
|
|
||||||
kind: "stt-online",
|
|
||||||
archiveUrl:
|
|
||||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2",
|
|
||||||
extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en",
|
|
||||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"],
|
|
||||||
description: "Streaming Paraformer (often strong accuracy; heavier).",
|
|
||||||
aliases: ["paraformer"],
|
|
||||||
},
|
|
||||||
"parakeet-tdt-0.6b-v2-int8": {
|
"parakeet-tdt-0.6b-v2-int8": {
|
||||||
kind: "stt-offline",
|
kind: "stt-offline",
|
||||||
archiveUrl:
|
archiveUrl:
|
||||||
@@ -46,27 +21,8 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
|
|||||||
extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
|
extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
|
||||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
|
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
|
||||||
description: "NVIDIA Parakeet TDT v2 (offline NeMo transducer, English).",
|
description: "NVIDIA Parakeet TDT v2 (offline NeMo transducer, English).",
|
||||||
aliases: ["parakeet-v2", "parakeet-tdt-v2"],
|
|
||||||
defaultFor: "stt",
|
defaultFor: "stt",
|
||||||
},
|
},
|
||||||
"parakeet-tdt-0.6b-v3-int8": {
|
|
||||||
kind: "stt-offline",
|
|
||||||
archiveUrl:
|
|
||||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
|
|
||||||
extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
|
|
||||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
|
|
||||||
description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).",
|
|
||||||
aliases: ["parakeet", "parakeet-v3", "parakeet-tdt"],
|
|
||||||
},
|
|
||||||
"kitten-nano-en-v0_1-fp16": {
|
|
||||||
kind: "tts",
|
|
||||||
archiveUrl:
|
|
||||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2",
|
|
||||||
extractedDir: "kitten-nano-en-v0_1-fp16",
|
|
||||||
requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
|
|
||||||
description: "KittenTTS (small, fast English TTS).",
|
|
||||||
aliases: ["kitten"],
|
|
||||||
},
|
|
||||||
"kokoro-en-v0_19": {
|
"kokoro-en-v0_19": {
|
||||||
kind: "tts",
|
kind: "tts",
|
||||||
archiveUrl:
|
archiveUrl:
|
||||||
@@ -74,54 +30,8 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
|
|||||||
extractedDir: "kokoro-en-v0_19",
|
extractedDir: "kokoro-en-v0_19",
|
||||||
requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
|
requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
|
||||||
description: "Kokoro TTS (higher quality; larger).",
|
description: "Kokoro TTS (higher quality; larger).",
|
||||||
aliases: ["kokoro"],
|
|
||||||
defaultFor: "tts",
|
defaultFor: "tts",
|
||||||
},
|
},
|
||||||
"pocket-tts-onnx-int8": {
|
|
||||||
kind: "tts",
|
|
||||||
extractedDir: "pocket-tts-onnx-int8",
|
|
||||||
downloadFiles: [
|
|
||||||
{
|
|
||||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_encoder.onnx",
|
|
||||||
relPath: "onnx/mimi_encoder.onnx",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/text_conditioner.onnx",
|
|
||||||
relPath: "onnx/text_conditioner.onnx",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_main_int8.onnx",
|
|
||||||
relPath: "onnx/flow_lm_main_int8.onnx",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_flow_int8.onnx",
|
|
||||||
relPath: "onnx/flow_lm_flow_int8.onnx",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_decoder_int8.onnx",
|
|
||||||
relPath: "onnx/mimi_decoder_int8.onnx",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/tokenizer.model",
|
|
||||||
relPath: "tokenizer.model",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/reference_sample.wav",
|
|
||||||
relPath: "reference_sample.wav",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
requiredFiles: [
|
|
||||||
"onnx/mimi_encoder.onnx",
|
|
||||||
"onnx/text_conditioner.onnx",
|
|
||||||
"onnx/flow_lm_main_int8.onnx",
|
|
||||||
"onnx/flow_lm_flow_int8.onnx",
|
|
||||||
"onnx/mimi_decoder_int8.onnx",
|
|
||||||
"tokenizer.model",
|
|
||||||
"reference_sample.wav",
|
|
||||||
],
|
|
||||||
description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).",
|
|
||||||
aliases: ["pocket", "pocket-tts"],
|
|
||||||
},
|
|
||||||
} as const satisfies Record<string, SherpaOnnxCatalogEntry>;
|
} as const satisfies Record<string, SherpaOnnxCatalogEntry>;
|
||||||
|
|
||||||
export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG;
|
export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG;
|
||||||
@@ -131,7 +41,7 @@ type ModelIdByKind<K extends SherpaOnnxModelKind> = {
|
|||||||
[Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K ? Id : never;
|
[Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K ? Id : never;
|
||||||
}[SherpaOnnxModelId];
|
}[SherpaOnnxModelId];
|
||||||
|
|
||||||
export type LocalSttModelId = ModelIdByKind<"stt-online"> | ModelIdByKind<"stt-offline">;
|
export type LocalSttModelId = ModelIdByKind<"stt-offline">;
|
||||||
export type LocalTtsModelId = ModelIdByKind<"tts">;
|
export type LocalTtsModelId = ModelIdByKind<"tts">;
|
||||||
|
|
||||||
const ALL_MODEL_IDS: SherpaOnnxModelId[] = Object.keys(SHERPA_ONNX_MODEL_CATALOG).filter(
|
const ALL_MODEL_IDS: SherpaOnnxModelId[] = Object.keys(SHERPA_ONNX_MODEL_CATALOG).filter(
|
||||||
@@ -166,48 +76,22 @@ function resolveDefaultModelId(role: DefaultModelRole): SherpaOnnxModelId {
|
|||||||
export const DEFAULT_LOCAL_STT_MODEL = resolveDefaultModelId("stt");
|
export const DEFAULT_LOCAL_STT_MODEL = resolveDefaultModelId("stt");
|
||||||
export const DEFAULT_LOCAL_TTS_MODEL = resolveDefaultModelId("tts");
|
export const DEFAULT_LOCAL_TTS_MODEL = resolveDefaultModelId("tts");
|
||||||
|
|
||||||
function buildAliasMap<T extends SherpaOnnxModelId>(modelIds: readonly T[]): Record<string, T> {
|
function createModelIdSchema<T extends string>(
|
||||||
const aliasMap: Record<string, T> = {};
|
modelIds: readonly T[],
|
||||||
for (const modelId of modelIds) {
|
): z.ZodType<T, z.ZodTypeDef, string> {
|
||||||
const aliases = SHERPA_ONNX_MODEL_CATALOG[modelId].aliases ?? [];
|
const validIds = new Set<string>(modelIds);
|
||||||
for (const alias of aliases) {
|
|
||||||
aliasMap[alias.trim().toLowerCase()] = modelId;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return aliasMap;
|
|
||||||
}
|
|
||||||
|
|
||||||
function createAliasedModelIdSchema<T extends string>(params: {
|
|
||||||
modelIds: readonly T[];
|
|
||||||
aliases: Record<string, T>;
|
|
||||||
}): z.ZodType<T, z.ZodTypeDef, string> {
|
|
||||||
const validIds = new Set<string>(params.modelIds);
|
|
||||||
return z
|
return z
|
||||||
.string()
|
.string()
|
||||||
.trim()
|
.trim()
|
||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
.refine(
|
.refine((value) => validIds.has(value), {
|
||||||
(value): value is T =>
|
message: "Invalid model id",
|
||||||
validIds.has(value) || Object.prototype.hasOwnProperty.call(params.aliases, value),
|
})
|
||||||
{
|
.transform((value) => value as T);
|
||||||
message: "Invalid model id",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.transform((value) => params.aliases[value] ?? value);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const STT_MODEL_ALIASES = buildAliasMap(LOCAL_STT_MODEL_IDS);
|
export const LocalSttModelIdSchema = createModelIdSchema(LOCAL_STT_MODEL_IDS);
|
||||||
const TTS_MODEL_ALIASES = buildAliasMap(LOCAL_TTS_MODEL_IDS);
|
export const LocalTtsModelIdSchema = createModelIdSchema(LOCAL_TTS_MODEL_IDS);
|
||||||
|
|
||||||
export const LocalSttModelIdSchema = createAliasedModelIdSchema({
|
|
||||||
modelIds: LOCAL_STT_MODEL_IDS,
|
|
||||||
aliases: STT_MODEL_ALIASES,
|
|
||||||
});
|
|
||||||
|
|
||||||
export const LocalTtsModelIdSchema = createAliasedModelIdSchema({
|
|
||||||
modelIds: LOCAL_TTS_MODEL_IDS,
|
|
||||||
aliases: TTS_MODEL_ALIASES,
|
|
||||||
});
|
|
||||||
|
|
||||||
export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & {
|
export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & {
|
||||||
id: SherpaOnnxModelId;
|
id: SherpaOnnxModelId;
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { describe, expect, test, vi } from "vitest";
|
import { describe, expect, test } from "vitest";
|
||||||
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
@@ -15,69 +15,27 @@ const logger = pino({ level: "silent" });
|
|||||||
describe("sherpa model downloader", () => {
|
describe("sherpa model downloader", () => {
|
||||||
test("getSherpaOnnxModelDir maps modelId to extractedDir", () => {
|
test("getSherpaOnnxModelDir maps modelId to extractedDir", () => {
|
||||||
const modelsDir = "/tmp/models";
|
const modelsDir = "/tmp/models";
|
||||||
expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8")).toContain(
|
expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8")).toContain(
|
||||||
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
|
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
|
||||||
);
|
|
||||||
expect(getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8")).toContain(
|
|
||||||
"pocket-tts-onnx-int8",
|
|
||||||
);
|
);
|
||||||
|
expect(getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19")).toContain("kokoro-en-v0_19");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ensureSherpaOnnxModel succeeds without downloading when files exist", async () => {
|
test("ensureSherpaOnnxModel succeeds without downloading when files exist", async () => {
|
||||||
const modelsDir = makeTmpDir();
|
const modelsDir = makeTmpDir();
|
||||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "kitten-nano-en-v0_1-fp16");
|
const modelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19");
|
||||||
|
|
||||||
mkdirSync(path.join(modelDir, "espeak-ng-data"), { recursive: true });
|
mkdirSync(path.join(modelDir, "espeak-ng-data"), { recursive: true });
|
||||||
writeFileSync(path.join(modelDir, "model.fp16.onnx"), "x");
|
writeFileSync(path.join(modelDir, "model.onnx"), "x");
|
||||||
writeFileSync(path.join(modelDir, "voices.bin"), "x");
|
writeFileSync(path.join(modelDir, "voices.bin"), "x");
|
||||||
writeFileSync(path.join(modelDir, "tokens.txt"), "x");
|
writeFileSync(path.join(modelDir, "tokens.txt"), "x");
|
||||||
|
|
||||||
const out = await ensureSherpaOnnxModel({
|
const out = await ensureSherpaOnnxModel({
|
||||||
modelsDir,
|
modelsDir,
|
||||||
modelId: "kitten-nano-en-v0_1-fp16",
|
modelId: "kokoro-en-v0_19",
|
||||||
logger,
|
logger,
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(out).toBe(modelDir);
|
expect(out).toBe(modelDir);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ensureSherpaOnnxModel logs lifecycle events without progress spam", async () => {
|
|
||||||
const modelsDir = makeTmpDir();
|
|
||||||
const infoMessages: string[] = [];
|
|
||||||
|
|
||||||
const loggerWithSpy = {
|
|
||||||
child: () => loggerWithSpy,
|
|
||||||
info: (_obj?: unknown, msg?: string) => {
|
|
||||||
if (typeof msg === "string") {
|
|
||||||
infoMessages.push(msg);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
error: () => undefined,
|
|
||||||
} as unknown as pino.Logger;
|
|
||||||
|
|
||||||
const originalFetch = globalThis.fetch;
|
|
||||||
const payload = Buffer.alloc(128 * 1024, 7);
|
|
||||||
const fetchMock = vi.fn(async () => {
|
|
||||||
return new Response(payload, {
|
|
||||||
status: 200,
|
|
||||||
headers: { "content-length": String(payload.length) },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
globalThis.fetch = fetchMock as typeof fetch;
|
|
||||||
|
|
||||||
try {
|
|
||||||
await ensureSherpaOnnxModel({
|
|
||||||
modelsDir,
|
|
||||||
modelId: "pocket-tts-onnx-int8",
|
|
||||||
logger: loggerWithSpy,
|
|
||||||
});
|
|
||||||
} finally {
|
|
||||||
globalThis.fetch = originalFetch;
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(fetchMock).toHaveBeenCalled();
|
|
||||||
expect(infoMessages).toContain("Starting model download");
|
|
||||||
expect(infoMessages).toContain("Model download completed");
|
|
||||||
expect(infoMessages).not.toContain("Downloading model artifact");
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -111,92 +111,55 @@ export async function ensureSherpaOnnxModel(
|
|||||||
logger.info({ modelsDir: options.modelsDir }, "Starting model download");
|
logger.info({ modelsDir: options.modelsDir }, "Starting model download");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (spec.archiveUrl) {
|
const downloadsDir = path.join(options.modelsDir, ".downloads");
|
||||||
const downloadsDir = path.join(options.modelsDir, ".downloads");
|
const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
|
||||||
const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
|
const archivePath = path.join(downloadsDir, archiveFilename);
|
||||||
const archivePath = path.join(downloadsDir, archiveFilename);
|
|
||||||
|
|
||||||
if (!(await isNonEmptyFile(archivePath))) {
|
if (!(await isNonEmptyFile(archivePath))) {
|
||||||
await downloadToFile({
|
await downloadToFile({
|
||||||
url: spec.archiveUrl,
|
url: spec.archiveUrl,
|
||||||
outputPath: archivePath,
|
outputPath: archivePath,
|
||||||
});
|
});
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
modelId: options.modelId,
|
|
||||||
archivePath,
|
|
||||||
modelDir,
|
|
||||||
},
|
|
||||||
"Extracting model archive",
|
|
||||||
);
|
|
||||||
await extractTarArchive(archivePath, options.modelsDir);
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
modelId: options.modelId,
|
|
||||||
modelDir,
|
|
||||||
},
|
|
||||||
"Verifying downloaded model files",
|
|
||||||
);
|
|
||||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
|
||||||
throw new Error(
|
|
||||||
`Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
modelId: options.modelId,
|
|
||||||
archivePath,
|
|
||||||
},
|
|
||||||
"Finalizing model artifacts",
|
|
||||||
);
|
|
||||||
try {
|
|
||||||
await rm(archivePath, { force: true });
|
|
||||||
} catch {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info({ modelDir }, "Model download completed");
|
|
||||||
return modelDir;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (spec.downloadFiles && spec.downloadFiles.length > 0) {
|
logger.info(
|
||||||
await mkdir(modelDir, { recursive: true });
|
{
|
||||||
|
modelId: options.modelId,
|
||||||
|
archivePath,
|
||||||
|
modelDir,
|
||||||
|
},
|
||||||
|
"Extracting model archive",
|
||||||
|
);
|
||||||
|
await extractTarArchive(archivePath, options.modelsDir);
|
||||||
|
|
||||||
await Promise.all(
|
logger.info(
|
||||||
spec.downloadFiles.map(async (file) => {
|
{
|
||||||
const dst = path.join(modelDir, file.relPath);
|
modelId: options.modelId,
|
||||||
if (await isNonEmptyFile(dst)) {
|
modelDir,
|
||||||
return;
|
},
|
||||||
}
|
"Verifying downloaded model files",
|
||||||
await downloadToFile({
|
);
|
||||||
url: file.url,
|
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
||||||
outputPath: dst,
|
throw new Error(
|
||||||
});
|
`Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`,
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
modelId: options.modelId,
|
|
||||||
modelDir,
|
|
||||||
},
|
|
||||||
"Verifying downloaded model files",
|
|
||||||
);
|
|
||||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
|
||||||
throw new Error(
|
|
||||||
`Downloaded files for ${options.modelId}, but required files are still missing in ${modelDir}.`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info({ modelDir }, "Model download completed");
|
|
||||||
return modelDir;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Model spec for ${options.modelId} has no archiveUrl or downloadFiles`);
|
logger.info(
|
||||||
|
{
|
||||||
|
modelId: options.modelId,
|
||||||
|
archivePath,
|
||||||
|
},
|
||||||
|
"Finalizing model artifacts",
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await rm(archivePath, { force: true });
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info({ modelDir }, "Model download completed");
|
||||||
|
return modelDir;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error({ err: error }, "Model download failed");
|
logger.error({ err: error }, "Model download failed");
|
||||||
throw error;
|
throw error;
|
||||||
|
|||||||
@@ -1,157 +0,0 @@
|
|||||||
import { existsSync } from "node:fs";
|
|
||||||
import type pino from "pino";
|
|
||||||
|
|
||||||
import { loadSherpaOnnx } from "./sherpa-onnx-loader.js";
|
|
||||||
|
|
||||||
export type SherpaOnlineRecognizerModel =
|
|
||||||
| {
|
|
||||||
kind: "transducer";
|
|
||||||
encoder: string;
|
|
||||||
decoder: string;
|
|
||||||
joiner: string;
|
|
||||||
tokens: string;
|
|
||||||
modelType?: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
kind: "paraformer";
|
|
||||||
encoder: string;
|
|
||||||
decoder: string;
|
|
||||||
tokens: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface SherpaOnlineRecognizerConfig {
|
|
||||||
model: SherpaOnlineRecognizerModel;
|
|
||||||
numThreads?: number;
|
|
||||||
provider?: "cpu";
|
|
||||||
debug?: 0 | 1;
|
|
||||||
sampleRate?: number;
|
|
||||||
featureDim?: number;
|
|
||||||
decodingMethod?: "greedy_search";
|
|
||||||
maxActivePaths?: number;
|
|
||||||
enableEndpoint?: 0 | 1;
|
|
||||||
rule1MinTrailingSilence?: number;
|
|
||||||
rule2MinTrailingSilence?: number;
|
|
||||||
rule3MinUtteranceLength?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
function assertFileExists(filePath: string, label: string): void {
|
|
||||||
if (!existsSync(filePath)) {
|
|
||||||
throw new Error(`Missing ${label}: ${filePath}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function validateModelFiles(model: SherpaOnlineRecognizerModel): void {
|
|
||||||
if (model.kind === "transducer") {
|
|
||||||
assertFileExists(model.encoder, "transducer encoder");
|
|
||||||
assertFileExists(model.decoder, "transducer decoder");
|
|
||||||
assertFileExists(model.joiner, "transducer joiner");
|
|
||||||
assertFileExists(model.tokens, "tokens");
|
|
||||||
} else {
|
|
||||||
assertFileExists(model.encoder, "paraformer encoder");
|
|
||||||
assertFileExists(model.decoder, "paraformer decoder");
|
|
||||||
assertFileExists(model.tokens, "tokens");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildModelConfig(model: SherpaOnlineRecognizerModel): object {
|
|
||||||
if (model.kind === "transducer") {
|
|
||||||
return {
|
|
||||||
transducer: {
|
|
||||||
encoder: model.encoder,
|
|
||||||
decoder: model.decoder,
|
|
||||||
joiner: model.joiner,
|
|
||||||
},
|
|
||||||
tokens: model.tokens,
|
|
||||||
modelType: model.modelType ?? "zipformer",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
paraformer: {
|
|
||||||
encoder: model.encoder,
|
|
||||||
decoder: model.decoder,
|
|
||||||
},
|
|
||||||
tokens: model.tokens,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SherpaOnlineStreamNative {
|
|
||||||
acceptWaveform: (sampleRate: number, samples: Float32Array) => void;
|
|
||||||
free?: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SherpaOnlineRecognizerNative {
|
|
||||||
config?: { featConfig?: { sampleRate?: number } };
|
|
||||||
createStream: () => SherpaOnlineStreamNative;
|
|
||||||
isReady: (stream: SherpaOnlineStreamNative) => boolean;
|
|
||||||
decode: (stream: SherpaOnlineStreamNative) => void;
|
|
||||||
getResult: (stream: SherpaOnlineStreamNative) => { text?: string } | string | undefined;
|
|
||||||
reset?: (stream: SherpaOnlineStreamNative) => void;
|
|
||||||
free?: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class SherpaOnlineRecognizerEngine {
|
|
||||||
public readonly recognizer: SherpaOnlineRecognizerNative;
|
|
||||||
public readonly sampleRate: number;
|
|
||||||
private readonly logger: pino.Logger;
|
|
||||||
|
|
||||||
constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) {
|
|
||||||
this.logger = logger.child({
|
|
||||||
module: "speech",
|
|
||||||
provider: "local",
|
|
||||||
component: "online-recognizer",
|
|
||||||
});
|
|
||||||
|
|
||||||
const { model } = config;
|
|
||||||
validateModelFiles(model);
|
|
||||||
|
|
||||||
const sherpa = loadSherpaOnnx();
|
|
||||||
const modelConfig = buildModelConfig(model);
|
|
||||||
|
|
||||||
const featConfig = {
|
|
||||||
sampleRate: config.sampleRate ?? 16000,
|
|
||||||
featureDim: config.featureDim ?? 80,
|
|
||||||
};
|
|
||||||
|
|
||||||
const recognizerConfig = {
|
|
||||||
featConfig,
|
|
||||||
modelConfig: {
|
|
||||||
...modelConfig,
|
|
||||||
// NOTE: In the WASM-backed `sherpa-onnx` npm package, online recognizers
|
|
||||||
// error when `numThreads > 1`. Keep the default conservative.
|
|
||||||
numThreads: config.numThreads ?? 1,
|
|
||||||
provider: config.provider ?? "cpu",
|
|
||||||
debug: config.debug ?? 0,
|
|
||||||
},
|
|
||||||
decodingMethod: config.decodingMethod ?? "greedy_search",
|
|
||||||
maxActivePaths: config.maxActivePaths ?? 4,
|
|
||||||
enableEndpoint: config.enableEndpoint ?? 0,
|
|
||||||
rule1MinTrailingSilence: config.rule1MinTrailingSilence ?? 2.4,
|
|
||||||
rule2MinTrailingSilence: config.rule2MinTrailingSilence ?? 1.2,
|
|
||||||
rule3MinUtteranceLength: config.rule3MinUtteranceLength ?? 20,
|
|
||||||
};
|
|
||||||
|
|
||||||
this.recognizer = sherpa.createOnlineRecognizer(
|
|
||||||
recognizerConfig,
|
|
||||||
) as SherpaOnlineRecognizerNative;
|
|
||||||
const sr = this.recognizer?.config?.featConfig?.sampleRate;
|
|
||||||
this.sampleRate =
|
|
||||||
typeof sr === "number" && Number.isFinite(sr) && sr > 0 ? sr : featConfig.sampleRate;
|
|
||||||
|
|
||||||
this.logger.info(
|
|
||||||
{ sampleRate: this.sampleRate, modelKind: model.kind, numThreads: config.numThreads ?? 2 },
|
|
||||||
"Sherpa online recognizer initialized",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
createStream(): SherpaOnlineStreamNative {
|
|
||||||
return this.recognizer.createStream();
|
|
||||||
}
|
|
||||||
|
|
||||||
free(): void {
|
|
||||||
try {
|
|
||||||
this.recognizer?.free?.();
|
|
||||||
} catch (err) {
|
|
||||||
this.logger.warn({ err }, "Failed to free sherpa recognizer");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
import { createRequire } from "node:module";
|
|
||||||
|
|
||||||
export interface SherpaOnnxModule {
|
|
||||||
createOnlineRecognizer: (config: unknown) => unknown;
|
|
||||||
createOfflineRecognizer: (config: unknown) => unknown;
|
|
||||||
createOfflineTts: (config: unknown) => unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
let cached: SherpaOnnxModule | null = null;
|
|
||||||
|
|
||||||
export function loadSherpaOnnx(): SherpaOnnxModule {
|
|
||||||
if (cached) {
|
|
||||||
return cached;
|
|
||||||
}
|
|
||||||
|
|
||||||
const require = createRequire(import.meta.url);
|
|
||||||
cached = require("sherpa-onnx") as SherpaOnnxModule;
|
|
||||||
return cached;
|
|
||||||
}
|
|
||||||
@@ -1,146 +0,0 @@
|
|||||||
import { EventEmitter } from "node:events";
|
|
||||||
import { v4 as uuidv4 } from "uuid";
|
|
||||||
|
|
||||||
import type { StreamingTranscriptionSession } from "../../../speech-provider.js";
|
|
||||||
import { pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js";
|
|
||||||
import {
|
|
||||||
SherpaOnlineRecognizerEngine,
|
|
||||||
type SherpaOnlineStreamNative,
|
|
||||||
} from "./sherpa-online-recognizer.js";
|
|
||||||
|
|
||||||
export class SherpaRealtimeTranscriptionSession
|
|
||||||
extends EventEmitter
|
|
||||||
implements StreamingTranscriptionSession
|
|
||||||
{
|
|
||||||
private readonly engine: SherpaOnlineRecognizerEngine;
|
|
||||||
private stream: SherpaOnlineStreamNative | null = null;
|
|
||||||
private connected = false;
|
|
||||||
|
|
||||||
public readonly requiredSampleRate: number;
|
|
||||||
private currentSegmentId: string | null = null;
|
|
||||||
private previousSegmentId: string | null = null;
|
|
||||||
private lastPartialText = "";
|
|
||||||
private readonly tailPaddingMs: number;
|
|
||||||
|
|
||||||
constructor(params: { engine: SherpaOnlineRecognizerEngine; tailPaddingMs?: number }) {
|
|
||||||
super();
|
|
||||||
this.engine = params.engine;
|
|
||||||
this.requiredSampleRate = this.engine.sampleRate;
|
|
||||||
this.tailPaddingMs = params.tailPaddingMs ?? 500;
|
|
||||||
}
|
|
||||||
|
|
||||||
async connect(): Promise<void> {
|
|
||||||
if (this.connected) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.stream = this.engine.createStream();
|
|
||||||
this.currentSegmentId = uuidv4();
|
|
||||||
this.connected = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
appendPcm16(pcm16le: Buffer): void {
|
|
||||||
if (!this.connected || !this.stream || !this.currentSegmentId) {
|
|
||||||
this.emit("error", new Error("Sherpa realtime session not connected"));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const peak = pcm16lePeakAbs(pcm16le);
|
|
||||||
const peakFloat = peak / 32768.0;
|
|
||||||
const targetPeak = 0.6;
|
|
||||||
const maxGain = 50;
|
|
||||||
const gain =
|
|
||||||
peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
|
|
||||||
const floatSamples = pcm16leToFloat32(pcm16le, gain);
|
|
||||||
this.stream.acceptWaveform(this.engine.sampleRate, floatSamples);
|
|
||||||
|
|
||||||
while (this.engine.recognizer.isReady(this.stream)) {
|
|
||||||
this.engine.recognizer.decode(this.stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
const rawResult = this.engine.recognizer.getResult(this.stream);
|
|
||||||
const text = (
|
|
||||||
(typeof rawResult === "object" && rawResult && "text" in rawResult
|
|
||||||
? rawResult.text
|
|
||||||
: undefined) ?? ""
|
|
||||||
).trim();
|
|
||||||
if (text !== this.lastPartialText) {
|
|
||||||
this.lastPartialText = text;
|
|
||||||
this.emit("transcript", {
|
|
||||||
segmentId: this.currentSegmentId,
|
|
||||||
transcript: text,
|
|
||||||
isFinal: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
commit(): void {
|
|
||||||
if (!this.connected || !this.stream || !this.currentSegmentId) {
|
|
||||||
this.emit("error", new Error("Sherpa realtime session not connected"));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const padSamples = Math.max(
|
|
||||||
0,
|
|
||||||
Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000),
|
|
||||||
);
|
|
||||||
if (padSamples > 0) {
|
|
||||||
this.stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
|
|
||||||
}
|
|
||||||
|
|
||||||
while (this.engine.recognizer.isReady(this.stream)) {
|
|
||||||
this.engine.recognizer.decode(this.stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
const rawFinal = this.engine.recognizer.getResult(this.stream);
|
|
||||||
const finalText = (
|
|
||||||
(typeof rawFinal === "object" && rawFinal && "text" in rawFinal
|
|
||||||
? rawFinal.text
|
|
||||||
: undefined) ?? ""
|
|
||||||
).trim();
|
|
||||||
const segmentId = this.currentSegmentId;
|
|
||||||
const previousSegmentId = this.previousSegmentId;
|
|
||||||
|
|
||||||
this.emit("committed", { segmentId, previousSegmentId });
|
|
||||||
this.emit("transcript", { segmentId, transcript: finalText, isFinal: true });
|
|
||||||
|
|
||||||
this.previousSegmentId = segmentId;
|
|
||||||
this.currentSegmentId = uuidv4();
|
|
||||||
this.lastPartialText = "";
|
|
||||||
this.engine.recognizer.reset?.(this.stream);
|
|
||||||
} catch (err) {
|
|
||||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
clear(): void {
|
|
||||||
if (!this.connected || !this.stream) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
this.engine.recognizer.reset?.(this.stream);
|
|
||||||
this.currentSegmentId = uuidv4();
|
|
||||||
this.lastPartialText = "";
|
|
||||||
} catch (err) {
|
|
||||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
close(): void {
|
|
||||||
if (!this.stream) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
this.stream.free?.();
|
|
||||||
} catch {
|
|
||||||
// ignore
|
|
||||||
} finally {
|
|
||||||
this.stream = null;
|
|
||||||
this.connected = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,184 +0,0 @@
|
|||||||
import { EventEmitter } from "node:events";
|
|
||||||
import { v4 as uuidv4 } from "uuid";
|
|
||||||
import type pino from "pino";
|
|
||||||
|
|
||||||
import type {
|
|
||||||
SpeechToTextProvider,
|
|
||||||
StreamingTranscriptionSession,
|
|
||||||
TranscriptionResult,
|
|
||||||
} from "../../../speech-provider.js";
|
|
||||||
import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
|
|
||||||
import {
|
|
||||||
parsePcm16MonoWav,
|
|
||||||
parsePcmRateFromFormat,
|
|
||||||
pcm16lePeakAbs,
|
|
||||||
pcm16leToFloat32,
|
|
||||||
} from "../../../audio.js";
|
|
||||||
import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
|
|
||||||
|
|
||||||
export interface SherpaSttConfig {
|
|
||||||
engine: SherpaOnlineRecognizerEngine;
|
|
||||||
silencePeakThreshold?: number;
|
|
||||||
tailPaddingMs?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class SherpaOnnxSTT implements SpeechToTextProvider {
|
|
||||||
private readonly engine: SherpaOnlineRecognizerEngine;
|
|
||||||
private readonly silencePeakThreshold: number;
|
|
||||||
private readonly tailPaddingMs: number;
|
|
||||||
private readonly logger: pino.Logger;
|
|
||||||
public readonly id = "local" as const;
|
|
||||||
|
|
||||||
constructor(config: SherpaSttConfig, logger: pino.Logger) {
|
|
||||||
this.engine = config.engine;
|
|
||||||
this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
|
|
||||||
this.tailPaddingMs = config.tailPaddingMs ?? 500;
|
|
||||||
this.logger = logger.child({ module: "speech", provider: "local", component: "stt" });
|
|
||||||
}
|
|
||||||
|
|
||||||
public createSession(params: {
|
|
||||||
logger: pino.Logger;
|
|
||||||
language?: string;
|
|
||||||
prompt?: string;
|
|
||||||
}): StreamingTranscriptionSession {
|
|
||||||
const emitter = new EventEmitter();
|
|
||||||
void params;
|
|
||||||
const requiredSampleRate = this.engine.sampleRate;
|
|
||||||
let connected = false;
|
|
||||||
let segmentId = uuidv4();
|
|
||||||
let previousSegmentId: string | null = null;
|
|
||||||
let pcm16: Buffer = Buffer.alloc(0);
|
|
||||||
|
|
||||||
return {
|
|
||||||
requiredSampleRate,
|
|
||||||
async connect() {
|
|
||||||
connected = true;
|
|
||||||
},
|
|
||||||
appendPcm16(chunk: Buffer) {
|
|
||||||
if (!connected) {
|
|
||||||
emitter.emit("error", new Error("STT session not connected"));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]);
|
|
||||||
},
|
|
||||||
commit: () => {
|
|
||||||
if (!connected) {
|
|
||||||
emitter.emit("error", new Error("STT session not connected"));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const committedId = segmentId;
|
|
||||||
const prev = previousSegmentId;
|
|
||||||
emitter.emit("committed", { segmentId: committedId, previousSegmentId: prev });
|
|
||||||
|
|
||||||
void (async () => {
|
|
||||||
try {
|
|
||||||
const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`);
|
|
||||||
emitter.emit("transcript", {
|
|
||||||
segmentId: committedId,
|
|
||||||
transcript: rt.text,
|
|
||||||
isFinal: true,
|
|
||||||
language: rt.language,
|
|
||||||
logprobs: rt.logprobs,
|
|
||||||
avgLogprob: rt.avgLogprob,
|
|
||||||
isLowConfidence: rt.isLowConfidence,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
emitter.emit("error", err);
|
|
||||||
} finally {
|
|
||||||
previousSegmentId = committedId;
|
|
||||||
segmentId = uuidv4();
|
|
||||||
pcm16 = Buffer.alloc(0);
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
},
|
|
||||||
clear() {
|
|
||||||
pcm16 = Buffer.alloc(0);
|
|
||||||
segmentId = uuidv4();
|
|
||||||
},
|
|
||||||
close() {
|
|
||||||
connected = false;
|
|
||||||
pcm16 = Buffer.alloc(0);
|
|
||||||
},
|
|
||||||
on(event: "committed" | "transcript" | "error", handler: (payload: never) => void) {
|
|
||||||
emitter.on(event, handler as (...args: unknown[]) => void);
|
|
||||||
return undefined;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
public async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
|
|
||||||
const start = Date.now();
|
|
||||||
|
|
||||||
let inputRate: number;
|
|
||||||
let pcm16: Buffer;
|
|
||||||
|
|
||||||
if (format.toLowerCase().includes("audio/wav")) {
|
|
||||||
const parsed = parsePcm16MonoWav(audioBuffer);
|
|
||||||
inputRate = parsed.sampleRate;
|
|
||||||
pcm16 = parsed.pcm16;
|
|
||||||
} else if (format.toLowerCase().includes("audio/pcm")) {
|
|
||||||
inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate;
|
|
||||||
pcm16 = audioBuffer;
|
|
||||||
} else {
|
|
||||||
throw new Error(`Unsupported audio format for sherpa STT: ${format}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const peak = pcm16lePeakAbs(pcm16);
|
|
||||||
if (peak < this.silencePeakThreshold) {
|
|
||||||
return { text: "", duration: Date.now() - start, isLowConfidence: true };
|
|
||||||
}
|
|
||||||
|
|
||||||
let pcmForModel = pcm16;
|
|
||||||
if (inputRate !== this.engine.sampleRate) {
|
|
||||||
const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate });
|
|
||||||
pcmForModel = resampler.processChunk(pcm16);
|
|
||||||
inputRate = this.engine.sampleRate;
|
|
||||||
}
|
|
||||||
|
|
||||||
const peakForModel = pcm16lePeakAbs(pcmForModel);
|
|
||||||
const peakFloat = peakForModel / 32768.0;
|
|
||||||
const targetPeak = 0.6;
|
|
||||||
const maxGain = 50;
|
|
||||||
const gain =
|
|
||||||
peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
|
|
||||||
|
|
||||||
const stream = this.engine.createStream();
|
|
||||||
try {
|
|
||||||
const floatSamples = pcm16leToFloat32(pcmForModel, gain);
|
|
||||||
stream.acceptWaveform(inputRate, floatSamples);
|
|
||||||
|
|
||||||
while (this.engine.recognizer.isReady(stream)) {
|
|
||||||
this.engine.recognizer.decode(stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
const padSamples = Math.max(
|
|
||||||
0,
|
|
||||||
Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000),
|
|
||||||
);
|
|
||||||
if (padSamples > 0) {
|
|
||||||
stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
|
|
||||||
}
|
|
||||||
|
|
||||||
while (this.engine.recognizer.isReady(stream)) {
|
|
||||||
this.engine.recognizer.decode(stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
const rawResult = this.engine.recognizer.getResult(stream);
|
|
||||||
const text = (
|
|
||||||
(typeof rawResult === "object" && rawResult && "text" in rawResult
|
|
||||||
? rawResult.text
|
|
||||||
: undefined) ?? ""
|
|
||||||
).trim();
|
|
||||||
const duration = Date.now() - start;
|
|
||||||
this.logger.debug({ duration, textLength: text.length }, "Sherpa transcription complete");
|
|
||||||
return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) };
|
|
||||||
} finally {
|
|
||||||
try {
|
|
||||||
stream.free?.();
|
|
||||||
} catch {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -6,7 +6,7 @@ import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-p
|
|||||||
import { chunkBuffer, float32ToPcm16le } from "../../../audio.js";
|
import { chunkBuffer, float32ToPcm16le } from "../../../audio.js";
|
||||||
import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js";
|
import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js";
|
||||||
|
|
||||||
export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16";
|
export type SherpaTtsPreset = "kokoro-en-v0_19";
|
||||||
|
|
||||||
export interface SherpaTtsConfig {
|
export interface SherpaTtsConfig {
|
||||||
preset: SherpaTtsPreset;
|
preset: SherpaTtsPreset;
|
||||||
@@ -41,9 +41,6 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
|||||||
private readonly logger: pino.Logger;
|
private readonly logger: pino.Logger;
|
||||||
|
|
||||||
constructor(config: SherpaTtsConfig, logger: pino.Logger) {
|
constructor(config: SherpaTtsConfig, logger: pino.Logger) {
|
||||||
if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
|
|
||||||
throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
|
|
||||||
}
|
|
||||||
this.logger = logger.child({ module: "speech", provider: "local", component: "tts" });
|
this.logger = logger.child({ module: "speech", provider: "local", component: "tts" });
|
||||||
this.speakerId = config.speakerId ?? 0;
|
this.speakerId = config.speakerId ?? 0;
|
||||||
this.speed = config.speed ?? 1.0;
|
this.speed = config.speed ?? 1.0;
|
||||||
@@ -53,8 +50,7 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
|||||||
throw new Error("sherpa-onnx-node OfflineTts is unavailable");
|
throw new Error("sherpa-onnx-node OfflineTts is unavailable");
|
||||||
}
|
}
|
||||||
|
|
||||||
const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx";
|
const modelPath = `${config.modelDir}/model.onnx`;
|
||||||
const modelPath = `${config.modelDir}/${modelFile}`;
|
|
||||||
const voicesPath = `${config.modelDir}/voices.bin`;
|
const voicesPath = `${config.modelDir}/voices.bin`;
|
||||||
const tokensPath = `${config.modelDir}/tokens.txt`;
|
const tokensPath = `${config.modelDir}/tokens.txt`;
|
||||||
const dataDir = `${config.modelDir}/espeak-ng-data`;
|
const dataDir = `${config.modelDir}/espeak-ng-data`;
|
||||||
@@ -64,26 +60,15 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
|||||||
assertFileExists(tokensPath, "TTS tokens");
|
assertFileExists(tokensPath, "TTS tokens");
|
||||||
assertFileExists(dataDir, "TTS espeak-ng dataDir");
|
assertFileExists(dataDir, "TTS espeak-ng dataDir");
|
||||||
|
|
||||||
const modelConfig =
|
const modelConfig = {
|
||||||
config.preset === "kokoro-en-v0_19"
|
kokoro: {
|
||||||
? {
|
model: modelPath,
|
||||||
kokoro: {
|
voices: voicesPath,
|
||||||
model: modelPath,
|
tokens: tokensPath,
|
||||||
voices: voicesPath,
|
dataDir,
|
||||||
tokens: tokensPath,
|
lengthScale: config.lengthScale ?? 1.0,
|
||||||
dataDir,
|
},
|
||||||
lengthScale: config.lengthScale ?? 1.0,
|
};
|
||||||
},
|
|
||||||
}
|
|
||||||
: {
|
|
||||||
kitten: {
|
|
||||||
model: modelPath,
|
|
||||||
voices: voicesPath,
|
|
||||||
tokens: tokensPath,
|
|
||||||
dataDir,
|
|
||||||
lengthScale: config.lengthScale ?? 1.0,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
const offlineTtsConfig = {
|
const offlineTtsConfig = {
|
||||||
model: modelConfig,
|
model: modelConfig,
|
||||||
|
|||||||
+37
-107
@@ -5,28 +5,15 @@ import path from "node:path";
|
|||||||
import pino from "pino";
|
import pino from "pino";
|
||||||
|
|
||||||
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
|
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
|
||||||
import type { SherpaOnnxModelId } from "./model-catalog.js";
|
|
||||||
import { createDaemonTestContext } from "../../../../test-utils/index.js";
|
import { createDaemonTestContext } from "../../../../test-utils/index.js";
|
||||||
import { parsePcm16MonoWav, wordSimilarity } from "../../../../test-utils/dictation-e2e.js";
|
import { parsePcm16MonoWav, wordSimilarity } from "../../../../test-utils/dictation-e2e.js";
|
||||||
import { SherpaOnnxTTS } from "./sherpa-tts.js";
|
import { SherpaOnnxTTS } from "./sherpa-tts.js";
|
||||||
import { PocketTtsOnnxTTS } from "../pocket/pocket-tts-onnx.js";
|
|
||||||
import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
|
|
||||||
import { SherpaOnnxSTT } from "./sherpa-stt.js";
|
|
||||||
import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
|
import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
|
||||||
import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js";
|
import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js";
|
||||||
|
|
||||||
const RUN = process.env.PASEO_SPEECH_E2E_DOWNLOAD === "1";
|
const RUN = process.env.PASEO_SPEECH_E2E_DOWNLOAD === "1";
|
||||||
const downloadTest = RUN ? test : test.skip;
|
const downloadTest = RUN ? test : test.skip;
|
||||||
|
|
||||||
type ModelSet = "zipformer-kitten" | "parakeet-pocket";
|
|
||||||
|
|
||||||
function getModelSet(): ModelSet {
|
|
||||||
const raw = (process.env.PASEO_SPEECH_E2E_MODEL_SET ?? "parakeet-pocket").trim().toLowerCase();
|
|
||||||
if (raw === "zipformer-kitten" || raw === "zipformer") return "zipformer-kitten";
|
|
||||||
if (raw === "parakeet-pocket" || raw === "parakeet") return "parakeet-pocket";
|
|
||||||
throw new Error(`Unknown PASEO_SPEECH_E2E_MODEL_SET: ${raw}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function readFixtureWav(): Promise<Buffer> {
|
async function readFixtureWav(): Promise<Buffer> {
|
||||||
const fixturePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.wav");
|
const fixturePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.wav");
|
||||||
return import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
|
return import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
|
||||||
@@ -92,27 +79,20 @@ function toAudioPcmFormat(format: string): string {
|
|||||||
const rest = trimmed.replace(/^pcm;?/i, "");
|
const rest = trimmed.replace(/^pcm;?/i, "");
|
||||||
return rest ? `audio/pcm;${rest}` : "audio/pcm";
|
return rest ? `audio/pcm;${rest}` : "audio/pcm";
|
||||||
}
|
}
|
||||||
// Fall back to treating it as a suffix (e.g. "rate=24000")
|
|
||||||
return `audio/pcm;${trimmed}`;
|
return `audio/pcm;${trimmed}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
downloadTest(
|
downloadTest(
|
||||||
"downloads models and runs streaming STT + streaming TTS",
|
"downloads default local speech models and runs STT + TTS",
|
||||||
async () => {
|
async () => {
|
||||||
const logger = pino({ level: "silent" });
|
const logger = pino({ level: "silent" });
|
||||||
const set = getModelSet();
|
|
||||||
|
|
||||||
const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-"));
|
const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-"));
|
||||||
const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech");
|
const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech");
|
||||||
|
|
||||||
const modelIds: SherpaOnnxModelId[] =
|
|
||||||
set === "parakeet-pocket"
|
|
||||||
? ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"]
|
|
||||||
: ["zipformer-bilingual-zh-en-2023-02-20", "kitten-nano-en-v0_1-fp16"];
|
|
||||||
|
|
||||||
await ensureSherpaOnnxModels({
|
await ensureSherpaOnnxModels({
|
||||||
modelsDir,
|
modelsDir,
|
||||||
modelIds,
|
modelIds: ["parakeet-tdt-0.6b-v2-int8", "kokoro-en-v0_19"],
|
||||||
logger,
|
logger,
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -128,16 +108,10 @@ downloadTest(
|
|||||||
local: {
|
local: {
|
||||||
modelsDir,
|
modelsDir,
|
||||||
models: {
|
models: {
|
||||||
dictationStt:
|
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||||
set === "parakeet-pocket"
|
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||||
? "parakeet-tdt-0.6b-v3-int8"
|
voiceTts: "kokoro-en-v0_19",
|
||||||
: "zipformer-bilingual-zh-en-2023-02-20",
|
voiceTtsSpeakerId: 0,
|
||||||
voiceStt:
|
|
||||||
set === "parakeet-pocket"
|
|
||||||
? "parakeet-tdt-0.6b-v3-int8"
|
|
||||||
: "zipformer-bilingual-zh-en-2023-02-20",
|
|
||||||
voiceTts:
|
|
||||||
set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -149,7 +123,6 @@ downloadTest(
|
|||||||
expect(sampleRate).toBe(16000);
|
expect(sampleRate).toBe(16000);
|
||||||
const format = "audio/pcm;rate=16000;bits=16";
|
const format = "audio/pcm;rate=16000;bits=16";
|
||||||
|
|
||||||
// Streaming STT: dictation path (verifies websocket streaming + partials + final)
|
|
||||||
const dictationId = `dict-download-${Date.now()}`;
|
const dictationId = `dict-download-${Date.now()}`;
|
||||||
let partialCount = 0;
|
let partialCount = 0;
|
||||||
const unsubscribe = ctx.client.on("dictation_stream_partial", (message) => {
|
const unsubscribe = ctx.client.on("dictation_stream_partial", (message) => {
|
||||||
@@ -159,15 +132,14 @@ downloadTest(
|
|||||||
});
|
});
|
||||||
|
|
||||||
await ctx.client.startDictationStream(dictationId, format);
|
await ctx.client.startDictationStream(dictationId, format);
|
||||||
const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
|
const chunkBytes = 3200;
|
||||||
let seq = 0;
|
let seq = 0;
|
||||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||||
ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
|
ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
|
||||||
seq += 1;
|
seq += 1;
|
||||||
}
|
}
|
||||||
const finalSeq = seq - 1;
|
const dictationFinal = await ctx.client.finishDictationStream(dictationId, seq - 1);
|
||||||
const dictationFinal = await ctx.client.finishDictationStream(dictationId, finalSeq);
|
|
||||||
unsubscribe();
|
unsubscribe();
|
||||||
|
|
||||||
expect(dictationFinal.text.toLowerCase()).toContain("voice note");
|
expect(dictationFinal.text.toLowerCase()).toContain("voice note");
|
||||||
@@ -175,7 +147,6 @@ downloadTest(
|
|||||||
expect(wordSimilarity(dictationFinal.text, baseline)).toBeGreaterThan(0.45);
|
expect(wordSimilarity(dictationFinal.text, baseline)).toBeGreaterThan(0.45);
|
||||||
expect(partialCount).toBeGreaterThan(0);
|
expect(partialCount).toBeGreaterThan(0);
|
||||||
|
|
||||||
// Voice-mode STT: chunked upload until isLast=true
|
|
||||||
const transcriptionPromise = waitForSignal<string>(30000, (resolve, reject) => {
|
const transcriptionPromise = waitForSignal<string>(30000, (resolve, reject) => {
|
||||||
const offResult = ctx.client.on("transcription_result", (message) => {
|
const offResult = ctx.client.on("transcription_result", (message) => {
|
||||||
if (message.type !== "transcription_result") return;
|
if (message.type !== "transcription_result") return;
|
||||||
@@ -218,80 +189,39 @@ downloadTest(
|
|||||||
await ctx.client.setVoiceMode(false);
|
await ctx.client.setVoiceMode(false);
|
||||||
rmSync(voiceCwd, { recursive: true, force: true });
|
rmSync(voiceCwd, { recursive: true, force: true });
|
||||||
|
|
||||||
// Streaming TTS: generate locally from downloaded model and validate chunking.
|
|
||||||
const ttsText = "This is a voice note.";
|
const ttsText = "This is a voice note.";
|
||||||
if (set === "parakeet-pocket") {
|
const ttsModelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19");
|
||||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8");
|
const tts = new SherpaOnnxTTS(
|
||||||
const tts = await PocketTtsOnnxTTS.create(
|
{ preset: "kokoro-en-v0_19", modelDir: ttsModelDir, speakerId: 0 },
|
||||||
{ modelDir, precision: "int8", targetChunkMs: 50 },
|
logger,
|
||||||
logger,
|
);
|
||||||
);
|
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
||||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
const { chunks, combined } = await readAllChunks(stream);
|
||||||
const { chunks, combined } = await readAllChunks(stream);
|
tts.free();
|
||||||
|
|
||||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
expect(ttsFormat).toMatch(/rate=\d+/);
|
||||||
expect(chunks.length).toBeGreaterThan(3);
|
expect(chunks.length).toBeGreaterThan(3);
|
||||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
expect(combined.byteLength).toBeGreaterThan(2000);
|
||||||
|
|
||||||
// Round trip: TTS -> STT (offline parakeet)
|
const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8");
|
||||||
const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8");
|
const engine = new SherpaOfflineRecognizerEngine(
|
||||||
const engine = new SherpaOfflineRecognizerEngine(
|
{
|
||||||
{
|
model: {
|
||||||
model: {
|
kind: "nemo_transducer",
|
||||||
kind: "nemo_transducer",
|
encoder: `${sttModelDir}/encoder.int8.onnx`,
|
||||||
encoder: `${sttModelDir}/encoder.int8.onnx`,
|
decoder: `${sttModelDir}/decoder.int8.onnx`,
|
||||||
decoder: `${sttModelDir}/decoder.int8.onnx`,
|
joiner: `${sttModelDir}/joiner.int8.onnx`,
|
||||||
joiner: `${sttModelDir}/joiner.int8.onnx`,
|
tokens: `${sttModelDir}/tokens.txt`,
|
||||||
tokens: `${sttModelDir}/tokens.txt`,
|
|
||||||
},
|
|
||||||
numThreads: 2,
|
|
||||||
debug: 0,
|
|
||||||
},
|
},
|
||||||
logger,
|
numThreads: 2,
|
||||||
);
|
debug: 0,
|
||||||
const stt = new SherpaOnnxParakeetSTT({ engine }, logger);
|
},
|
||||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
logger,
|
||||||
engine.free();
|
);
|
||||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
const stt = new SherpaOnnxParakeetSTT({ engine }, logger);
|
||||||
} else {
|
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
||||||
const ttsModelDir = path.join(modelsDir, "kitten-nano-en-v0_1-fp16");
|
engine.free();
|
||||||
const tts = new SherpaOnnxTTS(
|
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
||||||
{ preset: "kitten-nano-en-v0_1-fp16", modelDir: ttsModelDir },
|
|
||||||
logger,
|
|
||||||
);
|
|
||||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
|
||||||
const { chunks, combined } = await readAllChunks(stream);
|
|
||||||
tts.free();
|
|
||||||
|
|
||||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
|
||||||
expect(chunks.length).toBeGreaterThan(3);
|
|
||||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
|
||||||
|
|
||||||
// Round trip: TTS -> STT (online zipformer, offline segment)
|
|
||||||
const sttModelDir = path.join(
|
|
||||||
modelsDir,
|
|
||||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
|
||||||
);
|
|
||||||
const engine = new SherpaOnlineRecognizerEngine(
|
|
||||||
{
|
|
||||||
model: {
|
|
||||||
kind: "transducer",
|
|
||||||
encoder: `${sttModelDir}/encoder-epoch-99-avg-1.onnx`,
|
|
||||||
decoder: `${sttModelDir}/decoder-epoch-99-avg-1.onnx`,
|
|
||||||
joiner: `${sttModelDir}/joiner-epoch-99-avg-1.onnx`,
|
|
||||||
tokens: `${sttModelDir}/tokens.txt`,
|
|
||||||
modelType: "zipformer",
|
|
||||||
},
|
|
||||||
numThreads: 1,
|
|
||||||
debug: 0,
|
|
||||||
},
|
|
||||||
logger,
|
|
||||||
);
|
|
||||||
const stt = new SherpaOnnxSTT({ engine }, logger);
|
|
||||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
|
||||||
engine.free();
|
|
||||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
|
||||||
}
|
|
||||||
} finally {
|
} finally {
|
||||||
await ctx.cleanup();
|
await ctx.cleanup();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ describe("resolveSpeechConfig", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("resolves feature-scoped local model env vars", () => {
|
test("resolves feature-scoped local speech settings", () => {
|
||||||
const persisted = PersistedConfigSchema.parse({
|
const persisted = PersistedConfigSchema.parse({
|
||||||
features: {
|
features: {
|
||||||
voiceMode: {
|
voiceMode: {
|
||||||
@@ -70,9 +70,9 @@ describe("resolveSpeechConfig", () => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
const env = {
|
const env = {
|
||||||
PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer",
|
PASEO_DICTATION_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
|
||||||
PASEO_VOICE_LOCAL_STT_MODEL: "parakeet",
|
PASEO_VOICE_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
|
||||||
PASEO_VOICE_LOCAL_TTS_MODEL: "kitten",
|
PASEO_VOICE_LOCAL_TTS_MODEL: "kokoro-en-v0_19",
|
||||||
PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5",
|
PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5",
|
||||||
PASEO_VOICE_LOCAL_TTS_SPEED: "1.35",
|
PASEO_VOICE_LOCAL_TTS_SPEED: "1.35",
|
||||||
PASEO_DICTATION_LANGUAGE: "es",
|
PASEO_DICTATION_LANGUAGE: "es",
|
||||||
@@ -93,9 +93,9 @@ describe("resolveSpeechConfig", () => {
|
|||||||
expect(result.speech.local).toEqual({
|
expect(result.speech.local).toEqual({
|
||||||
modelsDir: "/tmp/models",
|
modelsDir: "/tmp/models",
|
||||||
models: {
|
models: {
|
||||||
dictationStt: "zipformer-bilingual-zh-en-2023-02-20",
|
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||||
voiceStt: "parakeet-tdt-0.6b-v3-int8",
|
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||||
voiceTts: "kitten-nano-en-v0_1-fp16",
|
voiceTts: "kokoro-en-v0_19",
|
||||||
voiceTtsSpeakerId: 5,
|
voiceTtsSpeakerId: 5,
|
||||||
voiceTtsSpeed: 1.35,
|
voiceTtsSpeed: 1.35,
|
||||||
},
|
},
|
||||||
@@ -120,9 +120,9 @@ describe("resolveSpeechConfig", () => {
|
|||||||
explicit: true,
|
explicit: true,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
});
|
});
|
||||||
expect(result.speech.local?.models.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20");
|
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||||
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
|
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||||
expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16");
|
expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
|
||||||
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5);
|
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5);
|
||||||
expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35);
|
expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35);
|
||||||
expect(result.speech.sttLanguages).toEqual({
|
expect(result.speech.sttLanguages).toEqual({
|
||||||
@@ -164,25 +164,6 @@ describe("resolveSpeechConfig", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ignores deprecated shared local model env vars", () => {
|
|
||||||
const persisted = PersistedConfigSchema.parse({});
|
|
||||||
const env = {
|
|
||||||
PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20",
|
|
||||||
PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16",
|
|
||||||
} as NodeJS.ProcessEnv;
|
|
||||||
|
|
||||||
const result = resolveSpeechConfig({
|
|
||||||
paseoHome: "/tmp/paseo-home",
|
|
||||||
env,
|
|
||||||
persisted,
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
|
||||||
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
|
||||||
expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
|
|
||||||
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("respects disabled dictation and voice mode feature flags", () => {
|
test("respects disabled dictation and voice mode feature flags", () => {
|
||||||
const persisted = PersistedConfigSchema.parse({
|
const persisted = PersistedConfigSchema.parse({
|
||||||
features: {
|
features: {
|
||||||
|
|||||||
@@ -293,8 +293,8 @@ function createReadySpeechReadinessSnapshot(): SpeechReadinessSnapshot {
|
|||||||
function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnapshot {
|
function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnapshot {
|
||||||
return {
|
return {
|
||||||
generatedAt: "2026-02-14T00:00:00.000Z",
|
generatedAt: "2026-02-14T00:00:00.000Z",
|
||||||
requiredLocalModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
|
requiredLocalModelIds: ["parakeet-tdt-0.6b-v2-int8"],
|
||||||
missingLocalModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
|
missingLocalModelIds: ["parakeet-tdt-0.6b-v2-int8"],
|
||||||
download: {
|
download: {
|
||||||
inProgress: true,
|
inProgress: true,
|
||||||
error: null,
|
error: null,
|
||||||
@@ -320,9 +320,9 @@ function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnaps
|
|||||||
available: false,
|
available: false,
|
||||||
reasonCode: "model_download_in_progress",
|
reasonCode: "model_download_in_progress",
|
||||||
message:
|
message:
|
||||||
"Voice features are unavailable while models download in the background (sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20).",
|
"Voice features are unavailable while models download in the background (parakeet-tdt-0.6b-v2-int8).",
|
||||||
retryable: true,
|
retryable: true,
|
||||||
missingModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
|
missingModelIds: ["parakeet-tdt-0.6b-v2-int8"],
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user