jb/harness-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dictate): add voice-reactive listening waveform

Browse Source
This commit is contained in:
jb
2026-08-26 13:40:47 -04:00
parent e234468a73
commit d1a811a5c5
16 changed files with 383 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/dictate/README.md
+8 -5
View File
@@ -61,12 +61,15 @@ The active profile is repository-linked on the tested host. Start a new Pi sandb
Shortcuts are handled by Pi's main editor. They do not claim to work while a selector, confirmation, or custom modal owns input. If a modal opens after recording starts, a successful result still goes only to the main editor.
The footer shows static service states:
The footer shows service state. While recording, a red circle and twelve-cell scrolling waveform make active capture visible and react to the microphone level:
- `loading model...`
- `listening...`
- `stopping...`
- `transcribing...`
```text
● listening ▁▂▄▇█▆▃▂▅▇▄▂
```
The circle uses Pi's semantic error color and the waveform uses its accent color. The other states remain `loading model...`, `stopping...`, and `transcribing...`. Audio levels are bounded scalar protocol events; raw audio remains inside the host service.
This indicator requires protocol version 2. Run the installer again after updating the repository so the installed host service and repository-linked extension upgrade together.
Only a non-empty finalized result for the current generation changes the editor. Empty speech, cancellation, stale results, disconnects, and failures do not insert text.
extensions/dictate/index.test.ts
+47 -3
View File
@@ -77,10 +77,17 @@ function createHarness() {
const sockets: FakeSocket[] = [];
const socketPaths: string[] = [];
const statuses: Array<string | undefined> = [];
const themeCalls: Array<{ color: string; text: string }> = [];
const notifications: Array<{ message: string; type?: string }> = [];
const pasted: string[] = [];
const context = {
ui: {
theme: {
fg: (color: string, text: string) => {
themeCalls.push({ color, text });
return text;
},
},
setStatus: (_key: string, text: string | undefined) => statuses.push(text),
notify: (message: string, type?: string) =>
notifications.push({ message, type }),
@@ -134,6 +141,7 @@ function createHarness() {
sockets,
socketPaths,
statuses,
themeCalls,
notifications,
pasted,
context,
@@ -213,7 +221,11 @@ test("rejects a mismatched protocol version before sending start", async () => {
assert.ok(socket);
socket.connect();
const hello = harness.sentFrames(socket)[0];
socket.frame({ version: 2, type: "pong", requestId: hello?.requestId });
socket.frame({
version: PROTOCOL_VERSION + 1,
type: "pong",
requestId: hello?.requestId,
});
assert.equal(harness.sentFrames(socket).length, 1);
assert.match(harness.notifications[0]?.message ?? "", /version mismatch/i);
@@ -266,19 +278,51 @@ test("renders authoritative service states and sends a bounded stop command", as
assert.deepEqual(harness.statuses, [
"loading model...",
"listening...",
"● listening ▁▁▁▁▁▁▁▁▁▁▁▁",
"stopping...",
"transcribing...",
]);
});
test("scrolls bounded audio levels through the listening waveform", async () => {
const harness = createHarness();
const { socket, generationId } = await connectAndNegotiate(harness);
serviceFrame(socket, generationId, "state", { state: "listening" });
assert.deepEqual(harness.themeCalls, [
{ color: "error", text: "●" },
{ color: "dim", text: "listening" },
{ color: "accent", text: "▁▁▁▁▁▁▁▁▁▁▁▁" },
]);
serviceFrame(socket, generationId, "level", { level: 0.5 });
serviceFrame(socket, generationId, "level", { level: 1 });
assert.deepEqual(harness.statuses, [
"● listening ▁▁▁▁▁▁▁▁▁▁▁▁",
"● listening ▁▁▁▁▁▁▁▁▁▁▁▅",
"● listening ▁▁▁▁▁▁▁▁▁▁▅█",
]);
});
test("rejects audio levels outside listening or the normalized range", async () => {
const beforeListening = createHarness();
const pending = await connectAndNegotiate(beforeListening);
serviceFrame(pending.socket, pending.generationId, "level", { level: 0.5 });
assert.match(beforeListening.notifications[0]?.message ?? "", /audio level is invalid/i);
const outOfRange = createHarness();
const active = await connectAndNegotiate(outOfRange);
serviceFrame(active.socket, active.generationId, "state", { state: "listening" });
serviceFrame(active.socket, active.generationId, "level", { level: 1.1 });
assert.match(outOfRange.notifications[0]?.message ?? "", /audio level is invalid/i);
});
test("ignores stale generations but rejects malformed and oversized frames", async () => {
const staleHarness = createHarness();
const { socket: staleSocket, generationId } = await connectAndNegotiate(staleHarness);
serviceFrame(staleSocket, "old-generation", "state", { state: "listening" });
assert.deepEqual(staleHarness.statuses, []);
serviceFrame(staleSocket, generationId, "state", { state: "listening" });
assert.deepEqual(staleHarness.statuses, ["listening..."]);
assert.deepEqual(staleHarness.statuses, ["● listening ▁▁▁▁▁▁▁▁▁▁▁▁"]);
const malformedHarness = createHarness();
const malformed = await connectAndNegotiate(malformedHarness);
extensions/dictate/index.ts
+43 -2
View File
@@ -2,7 +2,7 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-a
import { createConnection } from "node:net";
export const DICTATE_SOCKET_PATH = "/run/user/1000/pi-dictate.sock";
export const PROTOCOL_VERSION = 1;
export const PROTOCOL_VERSION = 2;
export const MAX_FRAME_BYTES = 64 * 1024;
export const MAX_TRANSCRIPT_BYTES = 32 * 1024;
export const MAX_ID_BYTES = 128;
@@ -10,6 +10,8 @@ export const MAX_ID_BYTES = 128;
const STATUS_KEY = "dictate";
const MAX_ERROR_DETAIL_BYTES = 2 * 1024;
const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true });
const WAVEFORM_BARS = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"] as const;
const WAVEFORM_SAMPLES = 12;
type DictateState =
| "idle"
@@ -97,6 +99,7 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
let receiveBuffer = Buffer.alloc(0);
let nextId = 0;
let resultDelivered = false;
let audioLevels: number[] = [];
const newId = (kind: string) => `${kind}-${++nextId}`;
@@ -104,6 +107,25 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
activeContext?.ui.setStatus(STATUS_KEY, status);
};
const setListeningStatus = () => {
if (!activeContext) return;
const theme = activeContext.ui.theme;
const waveform = audioLevels
.map(
(level) =>
WAVEFORM_BARS[
Math.min(
WAVEFORM_BARS.length - 1,
Math.floor(level * WAVEFORM_BARS.length),
)
],
)
.join("");
setStatus(
`${theme.fg("error", "●")} ${theme.fg("dim", "listening")} ${theme.fg("accent", waveform)}`,
);
};
const closeSocket = (current: DictateSocket, force: boolean) => {
try {
if (force) current.destroy();
@@ -121,6 +143,7 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
negotiated = false;
receiveBuffer = Buffer.alloc(0);
resultDelivered = false;
audioLevels = [];
state = "idle";
setStatus(undefined);
activeContext = null;
@@ -183,7 +206,8 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
}
if (serviceState === "listening" && state === "connecting") {
state = "recording";
setStatus("listening...");
audioLevels = Array<number>(WAVEFORM_SAMPLES).fill(0);
setListeningStatus();
return true;
}
if (serviceState === "stopping" && state === "stopping") {
@@ -267,6 +291,22 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
return;
}
if (value.type === "level") {
if (
state !== "recording" ||
typeof value.level !== "number" ||
!Number.isFinite(value.level) ||
value.level < 0 ||
value.level > 1
) {
protocolError("audio level is invalid for the current state.");
return;
}
audioLevels = [...audioLevels.slice(1), value.level];
setListeningStatus();
return;
}
if (value.type === "result") {
if (state !== "transcribing" || typeof value.transcript !== "string") {
protocolError("result is invalid for the current state.");
@@ -349,6 +389,7 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
helloRequestId = newId("hello");
negotiated = false;
resultDelivered = false;
audioLevels = [];
receiveBuffer = Buffer.alloc(0);
let current: DictateSocket;