feat(dictate): use local socket-activated Parakeet
This commit is contained in:
+374
-503
@@ -1,548 +1,419 @@
|
||||
/**
|
||||
* dictate — minimal voice dictation for pi.
|
||||
*
|
||||
* Press alt+m to start, press it again to stop.
|
||||
* Press alt+n to cancel and discard the in-flight transcript.
|
||||
*
|
||||
* Focus-aware: alt+m/alt+n are intercepted at the TUI input layer (before any
|
||||
* focused component), so dictation works inside ANY dialog — quiz popups,
|
||||
* ask_user_question, ctx.ui.editor()/input() — not just the main chat editor.
|
||||
*
|
||||
* Start rule: dictation only begins if some text-capable component is
|
||||
* focused; otherwise an ephemeral notification explains why nothing happened.
|
||||
* Opaque dialogs (quiz/ask selects) count as text-capable, but their internal
|
||||
* focus is invisible to us — Tab into the note/Other field first so the text
|
||||
* lands there.
|
||||
*
|
||||
* Stop rule: the delivery target is resolved fresh at stop time and the
|
||||
* transcript goes to whatever is focused THEN (editor-like components get a
|
||||
* direct setText append; opaque components get synthetic keystrokes). If
|
||||
* nothing text-capable is focused at stop, the transcript is copied to the
|
||||
* clipboard and a notification says so — a finished dictation is never lost.
|
||||
*
|
||||
* Requires:
|
||||
* - sox installed (`brew install sox` — provides the `rec` command)
|
||||
* - DEEPGRAM_API_KEY environment variable set
|
||||
*
|
||||
* Streaming model: audio is sent to Deepgram while you talk; the server
|
||||
* transcribes in real time and emits per-utterance "final" results. We
|
||||
* collect those finals and inject the concatenated text on stop. No
|
||||
* partials are shown in the editor (cosmetic-only), so quality is good
|
||||
* and the editor never shows revisable text.
|
||||
*/
|
||||
|
||||
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
||||
import { Key, matchesKey, isKeyRelease, isKeyRepeat } from "@earendil-works/pi-tui";
|
||||
import { spawn, type ChildProcessByStdio } from "node:child_process";
|
||||
import type { Readable } from "node:stream";
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { createConnection } from "node:net";
|
||||
|
||||
// Optional forensic logging: run pi with DICTATE_DEBUG=1 to append timestamped
|
||||
// lifecycle events (listener hits, toggles, ws open/error/close with their
|
||||
// generation) to /tmp/dictate-debug.log.
|
||||
const DEBUG = !!process.env.DICTATE_DEBUG;
|
||||
const dbg = (msg: string) => {
|
||||
if (!DEBUG) return;
|
||||
try {
|
||||
appendFileSync("/tmp/dictate-debug.log", `${new Date().toISOString()} ${msg}\n`);
|
||||
} catch {}
|
||||
};
|
||||
export const DICTATE_SOCKET_PATH = "/run/user/1000/pi-dictate.sock";
|
||||
export const PROTOCOL_VERSION = 1;
|
||||
export const MAX_FRAME_BYTES = 64 * 1024;
|
||||
export const MAX_TRANSCRIPT_BYTES = 32 * 1024;
|
||||
export const MAX_ID_BYTES = 128;
|
||||
|
||||
// Deepgram streaming endpoint. Tuning notes:
|
||||
// model=nova-3 — flagship, sub-300ms latency, best accuracy
|
||||
// encoding=linear16 — raw 16-bit PCM (what sox/rec gives us with -e signed-integer -b 16)
|
||||
// sample_rate=16000 — 16kHz mono is the standard low-bandwidth STT format
|
||||
// interim_results=false — we only want finals, never partials
|
||||
// smart_format=true — formats numbers, dates, currencies nicely
|
||||
// punctuate=true — adds commas/periods/question marks
|
||||
// endpointing=300 — 300ms of silence ends an utterance (faster finals)
|
||||
const DG_URL =
|
||||
"wss://api.deepgram.com/v1/listen" +
|
||||
"?model=nova-3" +
|
||||
"&encoding=linear16" +
|
||||
"&sample_rate=16000" +
|
||||
"&channels=1" +
|
||||
"&interim_results=false" +
|
||||
"&smart_format=true" +
|
||||
"&punctuate=true" +
|
||||
"&endpointing=300";
|
||||
const STATUS_KEY = "dictate";
|
||||
const MAX_ERROR_DETAIL_BYTES = 2 * 1024;
|
||||
const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true });
|
||||
|
||||
type State = "idle" | "recording" | "stopping";
|
||||
type DictateState =
|
||||
| "idle"
|
||||
| "connecting"
|
||||
| "recording"
|
||||
| "stopping"
|
||||
| "transcribing";
|
||||
type ServiceState =
|
||||
| "loading"
|
||||
| "listening"
|
||||
| "stopping"
|
||||
| "transcribing"
|
||||
| "idle";
|
||||
|
||||
// ── Focus-aware delivery ──────────────────────────────────────────────────
|
||||
// The TUI handle is captured once via a zero-height widget factory (the only
|
||||
// extension-API surface that exposes it). With it we can:
|
||||
// 1. Listen to ALL terminal input via tui.addInputListener — listeners run
|
||||
// before the focused component, so alt+m works even while a custom
|
||||
// dialog has stolen focus from the main editor (extension shortcuts are
|
||||
// otherwise only matched by the main editor component).
|
||||
// 2. Inspect tui.focusedComponent to decide where the transcript goes.
|
||||
// `focusedComponent` is declared private in the typings but is a plain
|
||||
// runtime property — a benign peek, easily patched if pi internals change.
|
||||
interface EditorLike {
|
||||
getText(): string;
|
||||
setText(text: string): void;
|
||||
export interface DictateSocket {
|
||||
onConnect(listener: () => void): void;
|
||||
onData(listener: (chunk: Uint8Array) => void): void;
|
||||
onError(listener: (error: Error) => void): void;
|
||||
onClose(listener: () => void): void;
|
||||
write(data: string): void;
|
||||
end(): void;
|
||||
destroy(): void;
|
||||
}
|
||||
type Target =
|
||||
| { kind: "editor"; editor: EditorLike }
|
||||
| { kind: "typable"; component: { handleInput(data: string): void } };
|
||||
|
||||
const asEditorLike = (value: any): EditorLike | null =>
|
||||
value && typeof value.getText === "function" && typeof value.setText === "function" ? value : null;
|
||||
export interface DictateExtensionOptions {
|
||||
socketFactory?: (path: string) => DictateSocket;
|
||||
socketPath?: string;
|
||||
}
|
||||
|
||||
// Same braille frames pi-tui's Loader uses.
|
||||
const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
|
||||
const SPINNER_INTERVAL_MS = 80;
|
||||
function nodeSocket(path: string): DictateSocket {
|
||||
const socket = createConnection(path);
|
||||
return {
|
||||
onConnect: (listener) => socket.on("connect", listener),
|
||||
onData: (listener) => socket.on("data", listener),
|
||||
onError: (listener) => socket.on("error", listener),
|
||||
onClose: (listener) => socket.on("close", listener),
|
||||
write: (data) => {
|
||||
socket.write(data);
|
||||
},
|
||||
end: () => {
|
||||
socket.end();
|
||||
},
|
||||
destroy: () => {
|
||||
socket.destroy();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Audio meter — a tiny rolling waveform rendered in the status row while recording.
|
||||
// Tweakable knobs:
|
||||
// METER_CELLS = how many bars wide
|
||||
// METER_TICK_MS = how often bars shift left (smaller = snappier, more renders)
|
||||
// METER_FLOOR_DB = level at which the bar is empty (more negative = more sensitive)
|
||||
// METER_CEILING_DB = level at which the bar is full (less negative = needs louder to peg)
|
||||
const METER_CELLS = 6;
|
||||
const METER_TICK_MS = 60;
|
||||
const PEAK_BLOCKS = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"];
|
||||
// const PEAK_BLOCKS = ["⠀", "⣀", "⣄", "⣤", "⣦", "⣶", "⣷", "⣿"];
|
||||
const METER_FLOOR_DB = -50;
|
||||
const METER_CEILING_DB = -10;
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** Compute normalized RMS (0..1) over a buffer of signed 16-bit little-endian PCM samples. */
|
||||
function rmsFromPcm16(buf: Buffer): number {
|
||||
const sampleCount = Math.floor(buf.length / 2);
|
||||
if (sampleCount === 0) return 0;
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < sampleCount * 2; i += 2) {
|
||||
const s = buf.readInt16LE(i);
|
||||
sumSquares += s * s;
|
||||
function utf8Bytes(value: string): number {
|
||||
return Buffer.byteLength(value, "utf8");
|
||||
}
|
||||
|
||||
function isBoundedId(value: unknown): value is string {
|
||||
return (
|
||||
typeof value === "string" &&
|
||||
value.length > 0 &&
|
||||
utf8Bytes(value) <= MAX_ID_BYTES
|
||||
);
|
||||
}
|
||||
|
||||
function boundedText(value: string, maxBytes: number): string {
|
||||
if (utf8Bytes(value) <= maxBytes) return value;
|
||||
let text = value;
|
||||
while (text.length > 0 && utf8Bytes(`${text}…`) > maxBytes) {
|
||||
text = text.slice(0, -1);
|
||||
}
|
||||
return Math.sqrt(sumSquares / sampleCount) / 32768;
|
||||
return `${text}…`;
|
||||
}
|
||||
|
||||
/** Map a normalized RMS value to one of PEAK_BLOCKS by converting to dB and clamping into the visible range. */
|
||||
function rmsToBlock(rms: number): string {
|
||||
if (rms <= 0) return PEAK_BLOCKS[0]!;
|
||||
const db = 20 * Math.log10(rms);
|
||||
const t = Math.max(0, Math.min(1, (db - METER_FLOOR_DB) / (METER_CEILING_DB - METER_FLOOR_DB)));
|
||||
const idx = Math.floor(t * (PEAK_BLOCKS.length - 1));
|
||||
return PEAK_BLOCKS[idx]!;
|
||||
}
|
||||
export function createDictateExtension(options: DictateExtensionOptions = {}) {
|
||||
const socketFactory = options.socketFactory ?? nodeSocket;
|
||||
const socketPath = options.socketPath ?? DICTATE_SOCKET_PATH;
|
||||
|
||||
export default function (pi: ExtensionAPI) {
|
||||
let state: State = "idle";
|
||||
let rec: ChildProcessByStdio<null, Readable, Readable> | null = null;
|
||||
let ws: WebSocket | null = null;
|
||||
let finals: string[] = [];
|
||||
let activeCtx: ExtensionContext | null = null;
|
||||
let flushed = false;
|
||||
let cancelled = false;
|
||||
let stopTimeout: NodeJS.Timeout | null = null;
|
||||
let spinnerTimer: NodeJS.Timeout | null = null;
|
||||
let spinnerFrame = 0;
|
||||
// Session generation: incremented on every start and every cleanup. All
|
||||
// rec/ws event handlers capture the generation they belong to and no-op
|
||||
// when it's stale — otherwise a PREVIOUS session's socket erroring/closing
|
||||
// late (e.g. one we aborted mid-handshake) would run cleanup() and tear
|
||||
// down the CURRENT live session.
|
||||
let generation = 0;
|
||||
// Audio meter state. `meter` is a ring of recent RMS values, newest at
|
||||
// index METER_CELLS-1. `currentLevel` is the most recent RMS observed from
|
||||
// any audio chunk — the meter tick just samples it. Crucially we never reset
|
||||
// it: empty ticks re-render the last observed value, so the bars never drop
|
||||
// to silence just because no chunk happened to arrive in that 60ms window.
|
||||
let meterTimer: NodeJS.Timeout | null = null;
|
||||
let meter: number[] = new Array(METER_CELLS).fill(0);
|
||||
let currentLevel = 0;
|
||||
return function registerDictate(pi: ExtensionAPI) {
|
||||
let state: DictateState = "idle";
|
||||
let socket: DictateSocket | null = null;
|
||||
let activeContext: ExtensionContext | null = null;
|
||||
let activeGenerationId: string | null = null;
|
||||
let helloRequestId: string | null = null;
|
||||
let negotiated = false;
|
||||
let receiveBuffer = Buffer.alloc(0);
|
||||
let nextId = 0;
|
||||
let resultDelivered = false;
|
||||
|
||||
const setStatus = (msg: string | undefined) => {
|
||||
if (!activeCtx) return;
|
||||
activeCtx.ui.setStatus("dictate", msg);
|
||||
};
|
||||
const newId = (kind: string) => `${kind}-${++nextId}`;
|
||||
|
||||
const stopSpinner = () => {
|
||||
if (spinnerTimer) {
|
||||
clearInterval(spinnerTimer);
|
||||
spinnerTimer = null;
|
||||
}
|
||||
};
|
||||
|
||||
const stopMeter = () => {
|
||||
if (meterTimer) {
|
||||
clearInterval(meterTimer);
|
||||
meterTimer = null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Start the meter ticking. Each tick shifts the ring and samples currentLevel. */
|
||||
const startMeter = () => {
|
||||
stopMeter();
|
||||
meter = new Array(METER_CELLS).fill(0);
|
||||
currentLevel = 0;
|
||||
// Recording dot: a text glyph colored via the theme, not an emoji — emoji
|
||||
// presentation renders double-width in its own baked-in color and visually
|
||||
// shouts in the footer. `●` is the same dot pi's own docs use for
|
||||
// indicators; theme "error" gives the red. (If you ever want strictly
|
||||
// ASCII, swap the glyph for "O".)
|
||||
const render = () => {
|
||||
const dot = activeCtx?.ui.theme.fg("error", "●") ?? "●";
|
||||
setStatus(`${dot} ${meter.map(rmsToBlock).join("")} listening…`);
|
||||
const setStatus = (status: string | undefined) => {
|
||||
activeContext?.ui.setStatus(STATUS_KEY, status);
|
||||
};
|
||||
render();
|
||||
meterTimer = setInterval(() => {
|
||||
meter.shift();
|
||||
meter.push(currentLevel);
|
||||
render();
|
||||
}, METER_TICK_MS);
|
||||
};
|
||||
|
||||
/** Animate the dictate status row with a braille spinner + suffix message. */
|
||||
const startSpinner = (suffix: string) => {
|
||||
stopSpinner();
|
||||
spinnerFrame = 0;
|
||||
setStatus(`${SPINNER_FRAMES[0]} ${suffix}`);
|
||||
spinnerTimer = setInterval(() => {
|
||||
spinnerFrame = (spinnerFrame + 1) % SPINNER_FRAMES.length;
|
||||
setStatus(`${SPINNER_FRAMES[spinnerFrame]} ${suffix}`);
|
||||
}, SPINNER_INTERVAL_MS);
|
||||
};
|
||||
|
||||
let tuiHandle: any = null;
|
||||
let removeInputListener: (() => void) | null = null;
|
||||
let lastCtx: ExtensionContext | null = null;
|
||||
|
||||
/** Resolve where dictated text would go RIGHT NOW, based on keyboard focus. */
|
||||
const resolveTarget = (): Target | null => {
|
||||
const focused = tuiHandle?.focusedComponent;
|
||||
if (!focused) return null;
|
||||
// Editor-like focus: the main chat editor, custom editors, and the
|
||||
// ctx.ui.editor()/input() popups (their inner pi-tui Editor hangs off
|
||||
// `.editor`). These accept a guaranteed direct setText append.
|
||||
const editor = asEditorLike(focused) ?? asEditorLike(focused.editor);
|
||||
if (editor) return { kind: "editor", editor };
|
||||
// Opaque component with input handling (quiz/ask selects, selectors):
|
||||
// we can type into it, but whether the text lands depends on its
|
||||
// internal focus (e.g. the quiz note field must be Tab-focused).
|
||||
if (typeof focused.handleInput === "function") return { kind: "typable", component: focused };
|
||||
return null;
|
||||
};
|
||||
|
||||
const flush = () => {
|
||||
if (flushed || !activeCtx) return;
|
||||
flushed = true;
|
||||
if (cancelled) return; // discard transcript on cancel
|
||||
const text = finals.join(" ").replace(/\s+/g, " ").trim();
|
||||
if (!text) return;
|
||||
|
||||
// Legacy fallback: no TUI handle captured (non-TUI mode / older pi) —
|
||||
// append to the main chat editor exactly as before.
|
||||
if (!tuiHandle) {
|
||||
const current = activeCtx.ui.getEditorText() ?? "";
|
||||
const sep = current && !/\s$/.test(current) ? " " : "";
|
||||
activeCtx.ui.setEditorText(current + sep + text);
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve the target NOW — focus may have changed while dictating.
|
||||
const target = resolveTarget();
|
||||
if (target?.kind === "editor") {
|
||||
const current = target.editor.getText() ?? "";
|
||||
const sep = current && !/\s$/.test(current) ? " " : "";
|
||||
target.editor.setText(current + sep + text);
|
||||
tuiHandle.requestRender?.();
|
||||
return;
|
||||
}
|
||||
if (target?.kind === "typable") {
|
||||
// Synthetic typing: the component routes the text wherever its
|
||||
// internal focus is. Text is plain printable words (whitespace
|
||||
// already normalized), so no keybindings/autocomplete can trigger.
|
||||
target.component.handleInput(text);
|
||||
tuiHandle.requestRender?.();
|
||||
return;
|
||||
}
|
||||
// Nothing to type into: don't throw the transcript away — stash it on
|
||||
// the clipboard and say so.
|
||||
try {
|
||||
const p = spawn("pbcopy", [], { stdio: ["pipe", "ignore", "ignore"] });
|
||||
p.stdin.end(text);
|
||||
} catch {}
|
||||
activeCtx.ui.notify("Dictation finished but no input field is focused — transcript copied to clipboard", "warning");
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
generation++; // invalidate the dying session's event handlers
|
||||
dbg(`cleanup → gen ${generation}`);
|
||||
flush();
|
||||
stopSpinner();
|
||||
stopMeter();
|
||||
if (stopTimeout) {
|
||||
clearTimeout(stopTimeout);
|
||||
stopTimeout = null;
|
||||
}
|
||||
if (rec) {
|
||||
const closeSocket = (current: DictateSocket, force: boolean) => {
|
||||
try {
|
||||
rec.kill("SIGTERM");
|
||||
} catch {}
|
||||
rec = null;
|
||||
}
|
||||
if (ws) {
|
||||
try {
|
||||
if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) {
|
||||
ws.close();
|
||||
}
|
||||
} catch {}
|
||||
ws = null;
|
||||
}
|
||||
finals = [];
|
||||
state = "idle";
|
||||
setStatus(undefined);
|
||||
activeCtx = null;
|
||||
flushed = false;
|
||||
cancelled = false;
|
||||
};
|
||||
if (force) current.destroy();
|
||||
else current.end();
|
||||
} catch {
|
||||
// Cleanup is best effort and must never deliver text.
|
||||
}
|
||||
};
|
||||
|
||||
const startDictation = (ctx: ExtensionContext) => {
|
||||
const apiKey = process.env.DEEPGRAM_API_KEY;
|
||||
if (!apiKey) {
|
||||
ctx.ui.notify("DEEPGRAM_API_KEY not set in environment", "error");
|
||||
return;
|
||||
}
|
||||
const reset = (forceClose = false) => {
|
||||
const current = socket;
|
||||
socket = null;
|
||||
activeGenerationId = null;
|
||||
helloRequestId = null;
|
||||
negotiated = false;
|
||||
receiveBuffer = Buffer.alloc(0);
|
||||
resultDelivered = false;
|
||||
state = "idle";
|
||||
setStatus(undefined);
|
||||
activeContext = null;
|
||||
if (current) closeSocket(current, forceClose);
|
||||
};
|
||||
|
||||
activeCtx = ctx;
|
||||
finals = [];
|
||||
flushed = false;
|
||||
cancelled = false;
|
||||
state = "recording";
|
||||
const myGeneration = ++generation;
|
||||
dbg(`start (gen ${myGeneration})`);
|
||||
startMeter();
|
||||
|
||||
// Spawn sox `rec` to capture 16kHz / 16-bit / mono PCM to stdout.
|
||||
let proc: ChildProcessByStdio<null, Readable, Readable>;
|
||||
try {
|
||||
proc = spawn(
|
||||
"rec",
|
||||
[
|
||||
"-q", // quiet
|
||||
// Shrink sox's IO buffer so stdout flushes ~every 16ms instead of
|
||||
// the default ~256ms. 512 bytes = 256 samples = 16ms at 16kHz/16-bit
|
||||
// mono. This is the dominant source of meter latency.
|
||||
"--buffer", "512",
|
||||
"-r", "16000",
|
||||
"-c", "1",
|
||||
"-b", "16",
|
||||
"-e", "signed-integer",
|
||||
"-t", "raw",
|
||||
"-", // stdout
|
||||
],
|
||||
{ stdio: ["ignore", "pipe", "pipe"] },
|
||||
const notifyError = (message: string) => {
|
||||
activeContext?.ui.notify(
|
||||
boundedText(message, MAX_ERROR_DETAIL_BYTES),
|
||||
"error",
|
||||
);
|
||||
} catch (e: any) {
|
||||
ctx.ui.notify(`Failed to spawn 'rec'. Install sox: brew install sox`, "error");
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
rec = proc;
|
||||
};
|
||||
|
||||
proc.on("error", (err) => {
|
||||
if (myGeneration !== generation) return;
|
||||
ctx.ui.notify(`rec error: ${err.message} (install sox: brew install sox)`, "error");
|
||||
cleanup();
|
||||
});
|
||||
const fail = (message: string) => {
|
||||
notifyError(message);
|
||||
reset(true);
|
||||
};
|
||||
|
||||
proc.on("exit", (code) => {
|
||||
if (myGeneration !== generation) return; // stale recorder — a newer/ended session owns state
|
||||
// Natural exit on SIGTERM during stopDictation is fine. Anything else
|
||||
// mid-recording is a problem.
|
||||
if (state === "recording" && code !== null && code !== 0) {
|
||||
if (activeCtx) {
|
||||
activeCtx.ui.notify(`rec exited unexpectedly (code ${code})`, "warning");
|
||||
}
|
||||
cleanup();
|
||||
const send = (frame: Record<string, string | number>): boolean => {
|
||||
if (!socket) return false;
|
||||
const encoded = `${JSON.stringify(frame)}\n`;
|
||||
if (utf8Bytes(encoded) - 1 > MAX_FRAME_BYTES) {
|
||||
fail("Dictation protocol error: outgoing frame is too large.");
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
// Open Deepgram WebSocket. Auth via subprotocol (portable across Node native
|
||||
// WebSocket and browsers): `new WebSocket(url, ["token", API_KEY])`.
|
||||
try {
|
||||
ws = new WebSocket(DG_URL, ["token", apiKey]);
|
||||
} catch (e: any) {
|
||||
ctx.ui.notify(`Deepgram WS failed: ${e.message}`, "error");
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
|
||||
ws.addEventListener("open", () => {
|
||||
if (myGeneration !== generation) {
|
||||
dbg(`ws open (stale gen ${myGeneration}, current ${generation}) — ignored`);
|
||||
return;
|
||||
try {
|
||||
socket.write(encoded);
|
||||
return true;
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : "write failed";
|
||||
fail(`Dictation service write failed: ${detail}`);
|
||||
return false;
|
||||
}
|
||||
dbg(`ws open (gen ${myGeneration})`);
|
||||
if (!rec || !ws) return;
|
||||
rec.stdout.on("data", (chunk: Buffer) => {
|
||||
// Track loudness for the meter (just the latest chunk's RMS — the meter
|
||||
// tick samples this), then forward to Deepgram.
|
||||
currentLevel = rmsFromPcm16(chunk);
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(chunk);
|
||||
}
|
||||
};
|
||||
|
||||
const sendGenerationCommand = (
|
||||
type: "start" | "stop" | "cancel",
|
||||
): boolean => {
|
||||
if (!activeGenerationId || !negotiated) return false;
|
||||
return send({
|
||||
version: PROTOCOL_VERSION,
|
||||
type,
|
||||
requestId: newId(type),
|
||||
generationId: activeGenerationId,
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
ws.addEventListener("message", (ev) => {
|
||||
if (myGeneration !== generation) return;
|
||||
try {
|
||||
const msg = JSON.parse(ev.data as string);
|
||||
if (msg.type === "Results" && msg.is_final) {
|
||||
const t = msg.channel?.alternatives?.[0]?.transcript;
|
||||
if (t) finals.push(t);
|
||||
const protocolError = (detail: string) => {
|
||||
fail(`Dictation protocol error: ${detail}`);
|
||||
};
|
||||
|
||||
const acceptServiceState = (serviceState: ServiceState): boolean => {
|
||||
if (serviceState === "idle") {
|
||||
reset();
|
||||
return true;
|
||||
}
|
||||
if (serviceState === "loading" && state === "connecting") {
|
||||
setStatus("loading model...");
|
||||
return true;
|
||||
}
|
||||
if (serviceState === "listening" && state === "connecting") {
|
||||
state = "recording";
|
||||
setStatus("listening...");
|
||||
return true;
|
||||
}
|
||||
if (serviceState === "stopping" && state === "stopping") {
|
||||
setStatus("stopping...");
|
||||
return true;
|
||||
}
|
||||
if (serviceState === "transcribing" && state === "stopping") {
|
||||
state = "transcribing";
|
||||
setStatus("transcribing...");
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
(serviceState === "loading" && state === "connecting") ||
|
||||
(serviceState === "listening" && state === "recording") ||
|
||||
(serviceState === "stopping" && state === "stopping") ||
|
||||
(serviceState === "transcribing" && state === "transcribing")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const handleFrame = (value: unknown, current: DictateSocket) => {
|
||||
if (socket !== current) return;
|
||||
if (!isRecord(value)) {
|
||||
protocolError("frame must be an object.");
|
||||
return;
|
||||
}
|
||||
if (value.version !== PROTOCOL_VERSION) {
|
||||
fail("Dictation service protocol version mismatch.");
|
||||
return;
|
||||
}
|
||||
if (typeof value.type !== "string" || !isBoundedId(value.requestId)) {
|
||||
protocolError("frame type or request ID is invalid.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.type === "pong") {
|
||||
if (
|
||||
negotiated ||
|
||||
state !== "connecting" ||
|
||||
value.requestId !== helloRequestId
|
||||
) {
|
||||
protocolError("unexpected hello response.");
|
||||
return;
|
||||
}
|
||||
// We could also handle msg.type === "Metadata" (sent after CloseStream
|
||||
// finishes draining), but ws.close handles the same flush path.
|
||||
} catch {
|
||||
// ignore non-JSON frames
|
||||
}
|
||||
});
|
||||
|
||||
ws.addEventListener("error", () => {
|
||||
if (myGeneration !== generation) {
|
||||
dbg(`ws error (stale gen ${myGeneration}, current ${generation}) — ignored`);
|
||||
negotiated = true;
|
||||
sendGenerationCommand("start");
|
||||
return;
|
||||
}
|
||||
dbg(`ws error (gen ${myGeneration})`);
|
||||
if (activeCtx) activeCtx.ui.notify("Deepgram WebSocket error", "error");
|
||||
cleanup();
|
||||
});
|
||||
|
||||
ws.addEventListener("close", (ev) => {
|
||||
if (myGeneration !== generation) {
|
||||
dbg(`ws close (stale gen ${myGeneration}, current ${generation}, code ${ev.code}) — ignored`);
|
||||
if (!negotiated) {
|
||||
protocolError("generation event received before hello negotiation.");
|
||||
return;
|
||||
}
|
||||
dbg(`ws close (gen ${myGeneration}, code ${ev.code})`);
|
||||
// Server-initiated close (or our own close in cleanup): finalize.
|
||||
if (state === "recording" || state === "stopping") {
|
||||
cleanup();
|
||||
|
||||
const generationId = value.generationId;
|
||||
if (!isBoundedId(generationId)) {
|
||||
protocolError("generation ID is invalid.");
|
||||
return;
|
||||
}
|
||||
});
|
||||
};
|
||||
if (generationId !== activeGenerationId) return;
|
||||
|
||||
/** Stop dictation, finalize transcript, append to editor. */
|
||||
const stopDictation = () => {
|
||||
if (state !== "recording") return;
|
||||
state = "stopping";
|
||||
stopMeter();
|
||||
startSpinner("finalizing…");
|
||||
if (value.type === "state") {
|
||||
const serviceState = value.state;
|
||||
if (
|
||||
typeof serviceState !== "string" ||
|
||||
![
|
||||
"loading",
|
||||
"listening",
|
||||
"stopping",
|
||||
"transcribing",
|
||||
"idle",
|
||||
].includes(serviceState)
|
||||
) {
|
||||
protocolError("service state is invalid.");
|
||||
return;
|
||||
}
|
||||
if (!acceptServiceState(serviceState as ServiceState)) {
|
||||
protocolError("service state transition is invalid.");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Stop the mic first so no more audio enqueues.
|
||||
if (rec) {
|
||||
if (value.type === "result") {
|
||||
if (state !== "transcribing" || typeof value.transcript !== "string") {
|
||||
protocolError("result is invalid for the current state.");
|
||||
return;
|
||||
}
|
||||
if (utf8Bytes(value.transcript) > MAX_TRANSCRIPT_BYTES) {
|
||||
protocolError("transcript is too large.");
|
||||
return;
|
||||
}
|
||||
if (resultDelivered) return;
|
||||
resultDelivered = true;
|
||||
const context = activeContext;
|
||||
const transcript = value.transcript;
|
||||
reset();
|
||||
if (transcript.trim().length === 0) {
|
||||
context?.ui.notify("No speech recognized.", "info");
|
||||
} else {
|
||||
context?.ui.pasteToEditor(transcript);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.type === "cancelled") {
|
||||
reset();
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.type === "error") {
|
||||
const code = typeof value.code === "string" ? value.code : "service_error";
|
||||
const detail = typeof value.detail === "string" ? value.detail : "Dictation failed";
|
||||
fail(`Dictation service error (${boundedText(code, 64)}): ${boundedText(detail, 384)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
protocolError("unknown frame type.");
|
||||
};
|
||||
|
||||
const consumeData = (chunk: Uint8Array, current: DictateSocket) => {
|
||||
if (socket !== current) return;
|
||||
receiveBuffer = Buffer.concat([receiveBuffer, Buffer.from(chunk)]);
|
||||
while (socket === current) {
|
||||
const newline = receiveBuffer.indexOf(0x0a);
|
||||
if (newline < 0) {
|
||||
if (receiveBuffer.length > MAX_FRAME_BYTES) {
|
||||
protocolError("frame is too large.");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (newline > MAX_FRAME_BYTES) {
|
||||
protocolError("frame is too large.");
|
||||
return;
|
||||
}
|
||||
const line = receiveBuffer.subarray(0, newline);
|
||||
receiveBuffer = receiveBuffer.subarray(newline + 1);
|
||||
if (line.length === 0) {
|
||||
protocolError("empty frame.");
|
||||
return;
|
||||
}
|
||||
let decoded: string;
|
||||
try {
|
||||
decoded = UTF8_DECODER.decode(line);
|
||||
} catch {
|
||||
protocolError("frame contains invalid UTF-8.");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
handleFrame(JSON.parse(decoded), current);
|
||||
} catch {
|
||||
protocolError("malformed JSON frame.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const startDictation = (ctx: ExtensionContext) => {
|
||||
if (state !== "idle") return;
|
||||
state = "connecting";
|
||||
activeContext = ctx;
|
||||
activeGenerationId = newId("generation");
|
||||
helloRequestId = newId("hello");
|
||||
negotiated = false;
|
||||
resultDelivered = false;
|
||||
receiveBuffer = Buffer.alloc(0);
|
||||
|
||||
let current: DictateSocket;
|
||||
try {
|
||||
rec.kill("SIGTERM");
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Tell Deepgram we're done; it will flush remaining finals then close.
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
ws.send(JSON.stringify({ type: "CloseStream" }));
|
||||
} catch {
|
||||
cleanup();
|
||||
current = socketFactory(socketPath);
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : "connection failed";
|
||||
fail(`Unable to connect to dictation service: ${detail}`);
|
||||
return;
|
||||
}
|
||||
// Safety net: if Deepgram never closes the socket, force cleanup after 3s.
|
||||
stopTimeout = setTimeout(() => {
|
||||
if (state === "stopping") cleanup();
|
||||
}, 3000);
|
||||
} else {
|
||||
cleanup();
|
||||
}
|
||||
};
|
||||
socket = current;
|
||||
current.onConnect(() => {
|
||||
if (socket !== current || !helloRequestId) return;
|
||||
send({
|
||||
version: PROTOCOL_VERSION,
|
||||
type: "hello",
|
||||
requestId: helloRequestId,
|
||||
});
|
||||
});
|
||||
current.onData((chunk) => consumeData(chunk, current));
|
||||
current.onError((error) => {
|
||||
if (socket !== current) return;
|
||||
fail(`Dictation service connection failed: ${error.message}`);
|
||||
});
|
||||
current.onClose(() => {
|
||||
if (socket !== current) return;
|
||||
fail("Dictation service disconnected.");
|
||||
});
|
||||
};
|
||||
|
||||
/** Cancel dictation: discard any collected transcript and tear everything down immediately. */
|
||||
const cancelDictation = () => {
|
||||
if (state !== "recording" && state !== "stopping") return;
|
||||
cancelled = true;
|
||||
finals = [];
|
||||
// No need to wait for Deepgram to flush — we're throwing the result away.
|
||||
cleanup();
|
||||
};
|
||||
const stopDictation = () => {
|
||||
if (state !== "recording") return;
|
||||
state = "stopping";
|
||||
sendGenerationCommand("stop");
|
||||
};
|
||||
|
||||
/** Toggle dictation, gated on there being somewhere for the text to go. */
|
||||
const toggleDictation = (ctx: ExtensionContext) => {
|
||||
lastCtx = ctx;
|
||||
if (state === "idle") {
|
||||
if (tuiHandle && !resolveTarget()) {
|
||||
ctx.ui.notify("No input field is focused — dictation not started", "warning");
|
||||
return;
|
||||
}
|
||||
startDictation(ctx);
|
||||
} else if (state === "recording") {
|
||||
stopDictation();
|
||||
}
|
||||
// Ignore presses during the "stopping" state — Deepgram is finalizing.
|
||||
};
|
||||
const cancelDictation = () => {
|
||||
if (state === "idle") return;
|
||||
if (negotiated) sendGenerationCommand("cancel");
|
||||
reset();
|
||||
};
|
||||
|
||||
// Global input listener: catches alt+m/alt+n before ANY focused component,
|
||||
// which is what makes dictation work inside dialogs. Registered once the
|
||||
// TUI handle is captured (see session_start below).
|
||||
const onGlobalInput = (data: string) => {
|
||||
// Kitty flag-2 terminals send press + REPEAT + RELEASE events, and input
|
||||
// listeners run BEFORE the TUI's release filter (that filter only guards
|
||||
// dispatch to the focused component). matchesKey also ignores the Kitty
|
||||
// event type. Without this guard a single physical alt+m press toggles
|
||||
// TWICE: press starts dictation, release instantly stops it and closes
|
||||
// the WebSocket mid-handshake — which then surfaces as
|
||||
// "Deepgram WebSocket error" (and its stale error event can kill the NEXT
|
||||
// session). Filter to press events only.
|
||||
if (isKeyRelease(data) || isKeyRepeat(data)) return undefined;
|
||||
if (matchesKey(data, Key.alt("m"))) {
|
||||
dbg(`alt+m (data=${JSON.stringify(data)}) state=${state}`);
|
||||
if (lastCtx) toggleDictation(lastCtx);
|
||||
return { consume: true };
|
||||
}
|
||||
if (matchesKey(data, Key.alt("n"))) {
|
||||
dbg(`alt+n (data=${JSON.stringify(data)}) state=${state}`);
|
||||
cancelDictation();
|
||||
return { consume: true };
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
const toggleDictation = (ctx: ExtensionContext) => {
|
||||
if (state === "idle") startDictation(ctx);
|
||||
else if (state === "recording") stopDictation();
|
||||
};
|
||||
|
||||
pi.on("session_start", (_event, ctx) => {
|
||||
lastCtx = ctx;
|
||||
if (ctx.mode !== "tui" || tuiHandle) return;
|
||||
// Capture the TUI handle via an invisible zero-height widget. The
|
||||
// listener function reference is stable, so even if the factory re-runs
|
||||
// the TUI's listener Set de-dupes it.
|
||||
ctx.ui.setWidget("dictate-tui-handle", (tui: any) => {
|
||||
tuiHandle = tui;
|
||||
removeInputListener = tui.addInputListener(onGlobalInput);
|
||||
return { render: () => [], invalidate: () => {} };
|
||||
pi.registerShortcut("alt+m", {
|
||||
description: "Start or stop local voice dictation",
|
||||
handler: async (ctx) => toggleDictation(ctx),
|
||||
});
|
||||
});
|
||||
pi.registerShortcut("alt+n", {
|
||||
description: "Cancel local voice dictation",
|
||||
handler: async () => cancelDictation(),
|
||||
});
|
||||
pi.registerCommand("dictate-cancel", {
|
||||
description: "Cancel local voice dictation",
|
||||
handler: async () => cancelDictation(),
|
||||
});
|
||||
pi.on("session_shutdown", async () => {
|
||||
if (state === "idle") return;
|
||||
if (negotiated) sendGenerationCommand("cancel");
|
||||
reset();
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
// Shortcut registrations kept as a fallback for contexts where the TUI
|
||||
// handle was never captured (non-TUI modes, older pi): they only fire when
|
||||
// the main editor is focused, but that's precisely the legacy path. When
|
||||
// the listener IS installed it consumes the key first, so no double-fire.
|
||||
pi.registerShortcut(Key.alt("m"), {
|
||||
description: "Toggle voice dictation (Deepgram)",
|
||||
handler: async (ctx) => {
|
||||
toggleDictation(ctx);
|
||||
},
|
||||
});
|
||||
|
||||
// Dedicated cancel binding. Dictation-only — a no-op when no dictation is
|
||||
// in flight, so it's safe to hammer without affecting anything else.
|
||||
pi.registerShortcut(Key.alt("n"), {
|
||||
description: "Cancel voice dictation (discard transcript)",
|
||||
handler: async () => {
|
||||
cancelDictation();
|
||||
},
|
||||
});
|
||||
|
||||
pi.on("session_shutdown", () => {
|
||||
if (state !== "idle") cleanup();
|
||||
removeInputListener?.();
|
||||
removeInputListener = null;
|
||||
});
|
||||
}
|
||||
export default createDictateExtension();
|
||||
|
||||
Reference in New Issue
Block a user