From c291ca083c616029dcb6d02287ad28af6fb7233c Mon Sep 17 00:00:00 2001 From: JB Date: Tue, 25 Aug 2026 22:53:36 -0400 Subject: [PATCH] Adding dictate extension --- extensions/dictate/LICENSE | 21 ++ extensions/dictate/index.ts | 548 ++++++++++++++++++++++++++++++++++++ 2 files changed, 569 insertions(+) create mode 100644 extensions/dictate/LICENSE create mode 100644 extensions/dictate/index.ts diff --git a/extensions/dictate/LICENSE b/extensions/dictate/LICENSE new file mode 100644 index 0000000..1749f92 --- /dev/null +++ b/extensions/dictate/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Amos Blomqvist + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/extensions/dictate/index.ts b/extensions/dictate/index.ts new file mode 100644 index 0000000..890d963 --- /dev/null +++ b/extensions/dictate/index.ts @@ -0,0 +1,548 @@ +/** + * dictate — minimal voice dictation for pi. + * + * Press alt+m to start, press it again to stop. + * Press alt+n to cancel and discard the in-flight transcript. + * + * Focus-aware: alt+m/alt+n are intercepted at the TUI input layer (before any + * focused component), so dictation works inside ANY dialog — quiz popups, + * ask_user_question, ctx.ui.editor()/input() — not just the main chat editor. + * + * Start rule: dictation only begins if some text-capable component is + * focused; otherwise an ephemeral notification explains why nothing happened. + * Opaque dialogs (quiz/ask selects) count as text-capable, but their internal + * focus is invisible to us — Tab into the note/Other field first so the text + * lands there. + * + * Stop rule: the delivery target is resolved fresh at stop time and the + * transcript goes to whatever is focused THEN (editor-like components get a + * direct setText append; opaque components get synthetic keystrokes). If + * nothing text-capable is focused at stop, the transcript is copied to the + * clipboard and a notification says so — a finished dictation is never lost. + * + * Requires: + * - sox installed (`brew install sox` — provides the `rec` command) + * - DEEPGRAM_API_KEY environment variable set + * + * Streaming model: audio is sent to Deepgram while you talk; the server + * transcribes in real time and emits per-utterance "final" results. We + * collect those finals and inject the concatenated text on stop. No + * partials are shown in the editor (cosmetic-only), so quality is good + * and the editor never shows revisable text. + */ + +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { Key, matchesKey, isKeyRelease, isKeyRepeat } from "@earendil-works/pi-tui"; +import { spawn, type ChildProcessByStdio } from "node:child_process"; +import type { Readable } from "node:stream"; +import { appendFileSync } from "node:fs"; + +// Optional forensic logging: run pi with DICTATE_DEBUG=1 to append timestamped +// lifecycle events (listener hits, toggles, ws open/error/close with their +// generation) to /tmp/dictate-debug.log. +const DEBUG = !!process.env.DICTATE_DEBUG; +const dbg = (msg: string) => { + if (!DEBUG) return; + try { + appendFileSync("/tmp/dictate-debug.log", `${new Date().toISOString()} ${msg}\n`); + } catch {} +}; + +// Deepgram streaming endpoint. Tuning notes: +// model=nova-3 — flagship, sub-300ms latency, best accuracy +// encoding=linear16 — raw 16-bit PCM (what sox/rec gives us with -e signed-integer -b 16) +// sample_rate=16000 — 16kHz mono is the standard low-bandwidth STT format +// interim_results=false — we only want finals, never partials +// smart_format=true — formats numbers, dates, currencies nicely +// punctuate=true — adds commas/periods/question marks +// endpointing=300 — 300ms of silence ends an utterance (faster finals) +const DG_URL = + "wss://api.deepgram.com/v1/listen" + + "?model=nova-3" + + "&encoding=linear16" + + "&sample_rate=16000" + + "&channels=1" + + "&interim_results=false" + + "&smart_format=true" + + "&punctuate=true" + + "&endpointing=300"; + +type State = "idle" | "recording" | "stopping"; + +// ── Focus-aware delivery ────────────────────────────────────────────────── +// The TUI handle is captured once via a zero-height widget factory (the only +// extension-API surface that exposes it). With it we can: +// 1. Listen to ALL terminal input via tui.addInputListener — listeners run +// before the focused component, so alt+m works even while a custom +// dialog has stolen focus from the main editor (extension shortcuts are +// otherwise only matched by the main editor component). +// 2. Inspect tui.focusedComponent to decide where the transcript goes. +// `focusedComponent` is declared private in the typings but is a plain +// runtime property — a benign peek, easily patched if pi internals change. +interface EditorLike { + getText(): string; + setText(text: string): void; +} +type Target = + | { kind: "editor"; editor: EditorLike } + | { kind: "typable"; component: { handleInput(data: string): void } }; + +const asEditorLike = (value: any): EditorLike | null => + value && typeof value.getText === "function" && typeof value.setText === "function" ? value : null; + +// Same braille frames pi-tui's Loader uses. +const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]; +const SPINNER_INTERVAL_MS = 80; + +// Audio meter — a tiny rolling waveform rendered in the status row while recording. +// Tweakable knobs: +// METER_CELLS = how many bars wide +// METER_TICK_MS = how often bars shift left (smaller = snappier, more renders) +// METER_FLOOR_DB = level at which the bar is empty (more negative = more sensitive) +// METER_CEILING_DB = level at which the bar is full (less negative = needs louder to peg) +const METER_CELLS = 6; +const METER_TICK_MS = 60; +const PEAK_BLOCKS = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"]; +// const PEAK_BLOCKS = ["⠀", "⣀", "⣄", "⣤", "⣦", "⣶", "⣷", "⣿"]; +const METER_FLOOR_DB = -50; +const METER_CEILING_DB = -10; + +/** Compute normalized RMS (0..1) over a buffer of signed 16-bit little-endian PCM samples. */ +function rmsFromPcm16(buf: Buffer): number { + const sampleCount = Math.floor(buf.length / 2); + if (sampleCount === 0) return 0; + let sumSquares = 0; + for (let i = 0; i < sampleCount * 2; i += 2) { + const s = buf.readInt16LE(i); + sumSquares += s * s; + } + return Math.sqrt(sumSquares / sampleCount) / 32768; +} + +/** Map a normalized RMS value to one of PEAK_BLOCKS by converting to dB and clamping into the visible range. */ +function rmsToBlock(rms: number): string { + if (rms <= 0) return PEAK_BLOCKS[0]!; + const db = 20 * Math.log10(rms); + const t = Math.max(0, Math.min(1, (db - METER_FLOOR_DB) / (METER_CEILING_DB - METER_FLOOR_DB))); + const idx = Math.floor(t * (PEAK_BLOCKS.length - 1)); + return PEAK_BLOCKS[idx]!; +} + +export default function (pi: ExtensionAPI) { + let state: State = "idle"; + let rec: ChildProcessByStdio | null = null; + let ws: WebSocket | null = null; + let finals: string[] = []; + let activeCtx: ExtensionContext | null = null; + let flushed = false; + let cancelled = false; + let stopTimeout: NodeJS.Timeout | null = null; + let spinnerTimer: NodeJS.Timeout | null = null; + let spinnerFrame = 0; + // Session generation: incremented on every start and every cleanup. All + // rec/ws event handlers capture the generation they belong to and no-op + // when it's stale — otherwise a PREVIOUS session's socket erroring/closing + // late (e.g. one we aborted mid-handshake) would run cleanup() and tear + // down the CURRENT live session. + let generation = 0; + // Audio meter state. `meter` is a ring of recent RMS values, newest at + // index METER_CELLS-1. `currentLevel` is the most recent RMS observed from + // any audio chunk — the meter tick just samples it. Crucially we never reset + // it: empty ticks re-render the last observed value, so the bars never drop + // to silence just because no chunk happened to arrive in that 60ms window. + let meterTimer: NodeJS.Timeout | null = null; + let meter: number[] = new Array(METER_CELLS).fill(0); + let currentLevel = 0; + + const setStatus = (msg: string | undefined) => { + if (!activeCtx) return; + activeCtx.ui.setStatus("dictate", msg); + }; + + const stopSpinner = () => { + if (spinnerTimer) { + clearInterval(spinnerTimer); + spinnerTimer = null; + } + }; + + const stopMeter = () => { + if (meterTimer) { + clearInterval(meterTimer); + meterTimer = null; + } + }; + + /** Start the meter ticking. Each tick shifts the ring and samples currentLevel. */ + const startMeter = () => { + stopMeter(); + meter = new Array(METER_CELLS).fill(0); + currentLevel = 0; + // Recording dot: a text glyph colored via the theme, not an emoji — emoji + // presentation renders double-width in its own baked-in color and visually + // shouts in the footer. `●` is the same dot pi's own docs use for + // indicators; theme "error" gives the red. (If you ever want strictly + // ASCII, swap the glyph for "O".) + const render = () => { + const dot = activeCtx?.ui.theme.fg("error", "●") ?? "●"; + setStatus(`${dot} ${meter.map(rmsToBlock).join("")} listening…`); + }; + render(); + meterTimer = setInterval(() => { + meter.shift(); + meter.push(currentLevel); + render(); + }, METER_TICK_MS); + }; + + /** Animate the dictate status row with a braille spinner + suffix message. */ + const startSpinner = (suffix: string) => { + stopSpinner(); + spinnerFrame = 0; + setStatus(`${SPINNER_FRAMES[0]} ${suffix}`); + spinnerTimer = setInterval(() => { + spinnerFrame = (spinnerFrame + 1) % SPINNER_FRAMES.length; + setStatus(`${SPINNER_FRAMES[spinnerFrame]} ${suffix}`); + }, SPINNER_INTERVAL_MS); + }; + + let tuiHandle: any = null; + let removeInputListener: (() => void) | null = null; + let lastCtx: ExtensionContext | null = null; + + /** Resolve where dictated text would go RIGHT NOW, based on keyboard focus. */ + const resolveTarget = (): Target | null => { + const focused = tuiHandle?.focusedComponent; + if (!focused) return null; + // Editor-like focus: the main chat editor, custom editors, and the + // ctx.ui.editor()/input() popups (their inner pi-tui Editor hangs off + // `.editor`). These accept a guaranteed direct setText append. + const editor = asEditorLike(focused) ?? asEditorLike(focused.editor); + if (editor) return { kind: "editor", editor }; + // Opaque component with input handling (quiz/ask selects, selectors): + // we can type into it, but whether the text lands depends on its + // internal focus (e.g. the quiz note field must be Tab-focused). + if (typeof focused.handleInput === "function") return { kind: "typable", component: focused }; + return null; + }; + + const flush = () => { + if (flushed || !activeCtx) return; + flushed = true; + if (cancelled) return; // discard transcript on cancel + const text = finals.join(" ").replace(/\s+/g, " ").trim(); + if (!text) return; + + // Legacy fallback: no TUI handle captured (non-TUI mode / older pi) — + // append to the main chat editor exactly as before. + if (!tuiHandle) { + const current = activeCtx.ui.getEditorText() ?? ""; + const sep = current && !/\s$/.test(current) ? " " : ""; + activeCtx.ui.setEditorText(current + sep + text); + return; + } + + // Resolve the target NOW — focus may have changed while dictating. + const target = resolveTarget(); + if (target?.kind === "editor") { + const current = target.editor.getText() ?? ""; + const sep = current && !/\s$/.test(current) ? " " : ""; + target.editor.setText(current + sep + text); + tuiHandle.requestRender?.(); + return; + } + if (target?.kind === "typable") { + // Synthetic typing: the component routes the text wherever its + // internal focus is. Text is plain printable words (whitespace + // already normalized), so no keybindings/autocomplete can trigger. + target.component.handleInput(text); + tuiHandle.requestRender?.(); + return; + } + // Nothing to type into: don't throw the transcript away — stash it on + // the clipboard and say so. + try { + const p = spawn("pbcopy", [], { stdio: ["pipe", "ignore", "ignore"] }); + p.stdin.end(text); + } catch {} + activeCtx.ui.notify("Dictation finished but no input field is focused — transcript copied to clipboard", "warning"); + }; + + const cleanup = () => { + generation++; // invalidate the dying session's event handlers + dbg(`cleanup → gen ${generation}`); + flush(); + stopSpinner(); + stopMeter(); + if (stopTimeout) { + clearTimeout(stopTimeout); + stopTimeout = null; + } + if (rec) { + try { + rec.kill("SIGTERM"); + } catch {} + rec = null; + } + if (ws) { + try { + if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) { + ws.close(); + } + } catch {} + ws = null; + } + finals = []; + state = "idle"; + setStatus(undefined); + activeCtx = null; + flushed = false; + cancelled = false; + }; + + const startDictation = (ctx: ExtensionContext) => { + const apiKey = process.env.DEEPGRAM_API_KEY; + if (!apiKey) { + ctx.ui.notify("DEEPGRAM_API_KEY not set in environment", "error"); + return; + } + + activeCtx = ctx; + finals = []; + flushed = false; + cancelled = false; + state = "recording"; + const myGeneration = ++generation; + dbg(`start (gen ${myGeneration})`); + startMeter(); + + // Spawn sox `rec` to capture 16kHz / 16-bit / mono PCM to stdout. + let proc: ChildProcessByStdio; + try { + proc = spawn( + "rec", + [ + "-q", // quiet + // Shrink sox's IO buffer so stdout flushes ~every 16ms instead of + // the default ~256ms. 512 bytes = 256 samples = 16ms at 16kHz/16-bit + // mono. This is the dominant source of meter latency. + "--buffer", "512", + "-r", "16000", + "-c", "1", + "-b", "16", + "-e", "signed-integer", + "-t", "raw", + "-", // stdout + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + } catch (e: any) { + ctx.ui.notify(`Failed to spawn 'rec'. Install sox: brew install sox`, "error"); + cleanup(); + return; + } + rec = proc; + + proc.on("error", (err) => { + if (myGeneration !== generation) return; + ctx.ui.notify(`rec error: ${err.message} (install sox: brew install sox)`, "error"); + cleanup(); + }); + + proc.on("exit", (code) => { + if (myGeneration !== generation) return; // stale recorder — a newer/ended session owns state + // Natural exit on SIGTERM during stopDictation is fine. Anything else + // mid-recording is a problem. + if (state === "recording" && code !== null && code !== 0) { + if (activeCtx) { + activeCtx.ui.notify(`rec exited unexpectedly (code ${code})`, "warning"); + } + cleanup(); + } + }); + + // Open Deepgram WebSocket. Auth via subprotocol (portable across Node native + // WebSocket and browsers): `new WebSocket(url, ["token", API_KEY])`. + try { + ws = new WebSocket(DG_URL, ["token", apiKey]); + } catch (e: any) { + ctx.ui.notify(`Deepgram WS failed: ${e.message}`, "error"); + cleanup(); + return; + } + + ws.addEventListener("open", () => { + if (myGeneration !== generation) { + dbg(`ws open (stale gen ${myGeneration}, current ${generation}) — ignored`); + return; + } + dbg(`ws open (gen ${myGeneration})`); + if (!rec || !ws) return; + rec.stdout.on("data", (chunk: Buffer) => { + // Track loudness for the meter (just the latest chunk's RMS — the meter + // tick samples this), then forward to Deepgram. + currentLevel = rmsFromPcm16(chunk); + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(chunk); + } + }); + }); + + ws.addEventListener("message", (ev) => { + if (myGeneration !== generation) return; + try { + const msg = JSON.parse(ev.data as string); + if (msg.type === "Results" && msg.is_final) { + const t = msg.channel?.alternatives?.[0]?.transcript; + if (t) finals.push(t); + } + // We could also handle msg.type === "Metadata" (sent after CloseStream + // finishes draining), but ws.close handles the same flush path. + } catch { + // ignore non-JSON frames + } + }); + + ws.addEventListener("error", () => { + if (myGeneration !== generation) { + dbg(`ws error (stale gen ${myGeneration}, current ${generation}) — ignored`); + return; + } + dbg(`ws error (gen ${myGeneration})`); + if (activeCtx) activeCtx.ui.notify("Deepgram WebSocket error", "error"); + cleanup(); + }); + + ws.addEventListener("close", (ev) => { + if (myGeneration !== generation) { + dbg(`ws close (stale gen ${myGeneration}, current ${generation}, code ${ev.code}) — ignored`); + return; + } + dbg(`ws close (gen ${myGeneration}, code ${ev.code})`); + // Server-initiated close (or our own close in cleanup): finalize. + if (state === "recording" || state === "stopping") { + cleanup(); + } + }); + }; + + /** Stop dictation, finalize transcript, append to editor. */ + const stopDictation = () => { + if (state !== "recording") return; + state = "stopping"; + stopMeter(); + startSpinner("finalizing…"); + + // Stop the mic first so no more audio enqueues. + if (rec) { + try { + rec.kill("SIGTERM"); + } catch {} + } + + // Tell Deepgram we're done; it will flush remaining finals then close. + if (ws && ws.readyState === WebSocket.OPEN) { + try { + ws.send(JSON.stringify({ type: "CloseStream" })); + } catch { + cleanup(); + return; + } + // Safety net: if Deepgram never closes the socket, force cleanup after 3s. + stopTimeout = setTimeout(() => { + if (state === "stopping") cleanup(); + }, 3000); + } else { + cleanup(); + } + }; + + /** Cancel dictation: discard any collected transcript and tear everything down immediately. */ + const cancelDictation = () => { + if (state !== "recording" && state !== "stopping") return; + cancelled = true; + finals = []; + // No need to wait for Deepgram to flush — we're throwing the result away. + cleanup(); + }; + + /** Toggle dictation, gated on there being somewhere for the text to go. */ + const toggleDictation = (ctx: ExtensionContext) => { + lastCtx = ctx; + if (state === "idle") { + if (tuiHandle && !resolveTarget()) { + ctx.ui.notify("No input field is focused — dictation not started", "warning"); + return; + } + startDictation(ctx); + } else if (state === "recording") { + stopDictation(); + } + // Ignore presses during the "stopping" state — Deepgram is finalizing. + }; + + // Global input listener: catches alt+m/alt+n before ANY focused component, + // which is what makes dictation work inside dialogs. Registered once the + // TUI handle is captured (see session_start below). + const onGlobalInput = (data: string) => { + // Kitty flag-2 terminals send press + REPEAT + RELEASE events, and input + // listeners run BEFORE the TUI's release filter (that filter only guards + // dispatch to the focused component). matchesKey also ignores the Kitty + // event type. Without this guard a single physical alt+m press toggles + // TWICE: press starts dictation, release instantly stops it and closes + // the WebSocket mid-handshake — which then surfaces as + // "Deepgram WebSocket error" (and its stale error event can kill the NEXT + // session). Filter to press events only. + if (isKeyRelease(data) || isKeyRepeat(data)) return undefined; + if (matchesKey(data, Key.alt("m"))) { + dbg(`alt+m (data=${JSON.stringify(data)}) state=${state}`); + if (lastCtx) toggleDictation(lastCtx); + return { consume: true }; + } + if (matchesKey(data, Key.alt("n"))) { + dbg(`alt+n (data=${JSON.stringify(data)}) state=${state}`); + cancelDictation(); + return { consume: true }; + } + return undefined; + }; + + pi.on("session_start", (_event, ctx) => { + lastCtx = ctx; + if (ctx.mode !== "tui" || tuiHandle) return; + // Capture the TUI handle via an invisible zero-height widget. The + // listener function reference is stable, so even if the factory re-runs + // the TUI's listener Set de-dupes it. + ctx.ui.setWidget("dictate-tui-handle", (tui: any) => { + tuiHandle = tui; + removeInputListener = tui.addInputListener(onGlobalInput); + return { render: () => [], invalidate: () => {} }; + }); + }); + + // Shortcut registrations kept as a fallback for contexts where the TUI + // handle was never captured (non-TUI modes, older pi): they only fire when + // the main editor is focused, but that's precisely the legacy path. When + // the listener IS installed it consumes the key first, so no double-fire. + pi.registerShortcut(Key.alt("m"), { + description: "Toggle voice dictation (Deepgram)", + handler: async (ctx) => { + toggleDictation(ctx); + }, + }); + + // Dedicated cancel binding. Dictation-only — a no-op when no dictation is + // in flight, so it's safe to hammer without affecting anything else. + pi.registerShortcut(Key.alt("n"), { + description: "Cancel voice dictation (discard transcript)", + handler: async () => { + cancelDictation(); + }, + }); + + pi.on("session_shutdown", () => { + if (state !== "idle") cleanup(); + removeInputListener?.(); + removeInputListener = null; + }); +} \ No newline at end of file