feat(dictate): support configurable ASR backends
This commit is contained in:
+228
-19
@@ -9,6 +9,7 @@ export const MAX_ID_BYTES = 128;
|
||||
|
||||
const STATUS_KEY = "dictate";
|
||||
const MAX_ERROR_DETAIL_BYTES = 2 * 1024;
|
||||
const MAX_DIAGNOSTIC_BYTES = 128;
|
||||
const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true });
|
||||
const WAVEFORM_BARS = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"] as const;
|
||||
const WAVEFORM_SAMPLES = 12;
|
||||
@@ -76,6 +77,15 @@ function isBoundedId(value: unknown): value is string {
|
||||
);
|
||||
}
|
||||
|
||||
function isBoundedDiagnostic(value: unknown): value is string {
|
||||
return (
|
||||
typeof value === "string" &&
|
||||
value.length > 0 &&
|
||||
utf8Bytes(value) <= MAX_DIAGNOSTIC_BYTES &&
|
||||
!/[\u0000-\u001f\u007f]/u.test(value)
|
||||
);
|
||||
}
|
||||
|
||||
function boundedText(value: string, maxBytes: number): string {
|
||||
if (utf8Bytes(value) <= maxBytes) return value;
|
||||
let text = value;
|
||||
@@ -92,14 +102,21 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
|
||||
return function registerDictate(pi: ExtensionAPI) {
|
||||
let state: DictateState = "idle";
|
||||
let socket: DictateSocket | null = null;
|
||||
let statusSocket: DictateSocket | null = null;
|
||||
let activeContext: ExtensionContext | null = null;
|
||||
let statusContext: ExtensionContext | null = null;
|
||||
let activeGenerationId: string | null = null;
|
||||
let helloRequestId: string | null = null;
|
||||
let negotiated = false;
|
||||
let receiveBuffer = Buffer.alloc(0);
|
||||
let receiveBuffer: Buffer<ArrayBufferLike> = Buffer.alloc(0);
|
||||
let nextId = 0;
|
||||
let resultDelivered = false;
|
||||
let audioLevels: number[] = [];
|
||||
let statusRequest: "hello" | "ping" | null = null;
|
||||
let statusRequestId: string | null = null;
|
||||
let statusBackend: string | null = null;
|
||||
let statusModel: string | null = null;
|
||||
let statusReceiveBuffer: Buffer<ArrayBufferLike> = Buffer.alloc(0);
|
||||
|
||||
const newId = (kind: string) => `${kind}-${++nextId}`;
|
||||
|
||||
@@ -135,7 +152,20 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
|
||||
}
|
||||
};
|
||||
|
||||
const closeStatusQuery = (forceClose = false) => {
|
||||
const current = statusSocket;
|
||||
statusSocket = null;
|
||||
statusContext = null;
|
||||
statusRequest = null;
|
||||
statusRequestId = null;
|
||||
statusBackend = null;
|
||||
statusModel = null;
|
||||
statusReceiveBuffer = Buffer.alloc(0);
|
||||
if (current) closeSocket(current, forceClose);
|
||||
};
|
||||
|
||||
const reset = (forceClose = false) => {
|
||||
closeStatusQuery(true);
|
||||
const current = socket;
|
||||
socket = null;
|
||||
activeGenerationId = null;
|
||||
@@ -162,6 +192,35 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
|
||||
reset(true);
|
||||
};
|
||||
|
||||
const failStatus = (message: string) => {
|
||||
const context = statusContext;
|
||||
closeStatusQuery(true);
|
||||
context?.ui.notify(
|
||||
boundedText(message, MAX_ERROR_DETAIL_BYTES),
|
||||
"error",
|
||||
);
|
||||
};
|
||||
|
||||
const sendStatus = (
|
||||
current: DictateSocket,
|
||||
frame: Record<string, string | number>,
|
||||
): boolean => {
|
||||
if (statusSocket !== current) return false;
|
||||
const encoded = `${JSON.stringify(frame)}\n`;
|
||||
if (utf8Bytes(encoded) - 1 > MAX_FRAME_BYTES) {
|
||||
failStatus("Dictation protocol error: outgoing frame is too large.");
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
current.write(encoded);
|
||||
return true;
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : "write failed";
|
||||
failStatus(`Dictation status request failed: ${detail}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const send = (frame: Record<string, string | number>): boolean => {
|
||||
if (!socket) return false;
|
||||
const encoded = `${JSON.stringify(frame)}\n`;
|
||||
@@ -201,7 +260,7 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
|
||||
return true;
|
||||
}
|
||||
if (serviceState === "loading" && state === "connecting") {
|
||||
setStatus("loading model...");
|
||||
setStatus("preparing transcription...");
|
||||
return true;
|
||||
}
|
||||
if (serviceState === "listening" && state === "connecting") {
|
||||
@@ -246,6 +305,12 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
|
||||
}
|
||||
|
||||
if (value.type === "pong") {
|
||||
for (const field of ["backend", "model"] as const) {
|
||||
if (field in value && !isBoundedDiagnostic(value[field])) {
|
||||
protocolError(`diagnostic ${field} is invalid.`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (
|
||||
negotiated ||
|
||||
state !== "connecting" ||
|
||||
@@ -344,45 +409,184 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
|
||||
protocolError("unknown frame type.");
|
||||
};
|
||||
|
||||
const consumeData = (chunk: Uint8Array, current: DictateSocket) => {
|
||||
if (socket !== current) return;
|
||||
receiveBuffer = Buffer.concat([receiveBuffer, Buffer.from(chunk)]);
|
||||
while (socket === current) {
|
||||
const newline = receiveBuffer.indexOf(0x0a);
|
||||
const handleStatusFrame = (value: unknown, current: DictateSocket) => {
|
||||
if (statusSocket !== current) return;
|
||||
if (!isRecord(value)) {
|
||||
failStatus("Dictation status protocol error: frame must be an object.");
|
||||
return;
|
||||
}
|
||||
if (value.version !== PROTOCOL_VERSION) {
|
||||
failStatus("Dictation status protocol version mismatch.");
|
||||
return;
|
||||
}
|
||||
if (typeof value.type !== "string" || !isBoundedId(value.requestId)) {
|
||||
failStatus("Dictation status protocol error: frame is invalid.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.type === "error") {
|
||||
const code = typeof value.code === "string" ? value.code : "service_error";
|
||||
const detail = typeof value.detail === "string" ? value.detail : "Dictation status failed";
|
||||
failStatus(
|
||||
`Dictation service error (${boundedText(code, 64)}): ${boundedText(detail, 384)}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (value.type !== "pong" || !statusRequestId || value.requestId !== statusRequestId) {
|
||||
failStatus("Dictation status protocol error: unexpected response.");
|
||||
return;
|
||||
}
|
||||
for (const field of ["backend", "model"] as const) {
|
||||
if (field in value && !isBoundedDiagnostic(value[field])) {
|
||||
failStatus(`Dictation status protocol error: diagnostic ${field} is invalid.`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (isBoundedDiagnostic(value.backend)) statusBackend = value.backend;
|
||||
if (isBoundedDiagnostic(value.model)) statusModel = value.model;
|
||||
|
||||
if (statusRequest === "hello") {
|
||||
statusRequest = "ping";
|
||||
statusRequestId = newId("status-ping");
|
||||
sendStatus(current, {
|
||||
version: PROTOCOL_VERSION,
|
||||
type: "ping",
|
||||
requestId: statusRequestId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (statusRequest !== "ping") {
|
||||
failStatus("Dictation status protocol error: status request is invalid.");
|
||||
return;
|
||||
}
|
||||
|
||||
const backend = statusBackend ?? "unknown";
|
||||
const model = statusModel ?? "unknown";
|
||||
const context = statusContext;
|
||||
closeStatusQuery();
|
||||
context?.ui.notify(
|
||||
`Dictation backend: ${backend}; model: ${model}`,
|
||||
"info",
|
||||
);
|
||||
};
|
||||
|
||||
const consumeFramedData = (
|
||||
chunk: Uint8Array,
|
||||
current: DictateSocket,
|
||||
isCurrent: () => boolean,
|
||||
getBuffer: () => Buffer<ArrayBufferLike>,
|
||||
setBuffer: (value: Buffer<ArrayBufferLike>) => void,
|
||||
onFrame: (value: unknown, socket: DictateSocket) => void,
|
||||
onProtocolError: (detail: string) => void,
|
||||
) => {
|
||||
if (!isCurrent()) return;
|
||||
let buffer = Buffer.concat([getBuffer(), Buffer.from(chunk)]);
|
||||
while (isCurrent()) {
|
||||
const newline = buffer.indexOf(0x0a);
|
||||
if (newline < 0) {
|
||||
if (receiveBuffer.length > MAX_FRAME_BYTES) {
|
||||
protocolError("frame is too large.");
|
||||
setBuffer(buffer);
|
||||
if (buffer.length > MAX_FRAME_BYTES) {
|
||||
onProtocolError("frame is too large.");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (newline > MAX_FRAME_BYTES) {
|
||||
protocolError("frame is too large.");
|
||||
setBuffer(buffer);
|
||||
onProtocolError("frame is too large.");
|
||||
return;
|
||||
}
|
||||
const line = receiveBuffer.subarray(0, newline);
|
||||
receiveBuffer = receiveBuffer.subarray(newline + 1);
|
||||
const line = buffer.subarray(0, newline);
|
||||
buffer = buffer.subarray(newline + 1);
|
||||
setBuffer(buffer);
|
||||
if (line.length === 0) {
|
||||
protocolError("empty frame.");
|
||||
onProtocolError("empty frame.");
|
||||
return;
|
||||
}
|
||||
let decoded: string;
|
||||
try {
|
||||
decoded = UTF8_DECODER.decode(line);
|
||||
} catch {
|
||||
protocolError("frame contains invalid UTF-8.");
|
||||
onProtocolError("frame contains invalid UTF-8.");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
handleFrame(JSON.parse(decoded), current);
|
||||
onFrame(JSON.parse(decoded), current);
|
||||
} catch {
|
||||
protocolError("malformed JSON frame.");
|
||||
onProtocolError("malformed JSON frame.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const consumeStatusData = (chunk: Uint8Array, current: DictateSocket) =>
|
||||
consumeFramedData(
|
||||
chunk,
|
||||
current,
|
||||
() => statusSocket === current,
|
||||
() => statusReceiveBuffer,
|
||||
(value) => {
|
||||
statusReceiveBuffer = value;
|
||||
},
|
||||
handleStatusFrame,
|
||||
(detail) => failStatus(`Dictation status protocol error: ${detail}`),
|
||||
);
|
||||
|
||||
const consumeData = (chunk: Uint8Array, current: DictateSocket) =>
|
||||
consumeFramedData(
|
||||
chunk,
|
||||
current,
|
||||
() => socket === current,
|
||||
() => receiveBuffer,
|
||||
(value) => {
|
||||
receiveBuffer = value;
|
||||
},
|
||||
handleFrame,
|
||||
protocolError,
|
||||
);
|
||||
|
||||
const statusDictation = (ctx: ExtensionContext) => {
|
||||
if (state !== "idle") {
|
||||
ctx.ui.notify("Dictation is active; status is unavailable.", "info");
|
||||
return;
|
||||
}
|
||||
if (statusSocket) return;
|
||||
|
||||
statusContext = ctx;
|
||||
statusRequest = "hello";
|
||||
statusRequestId = newId("status-hello");
|
||||
statusReceiveBuffer = Buffer.alloc(0);
|
||||
|
||||
let current: DictateSocket;
|
||||
try {
|
||||
current = socketFactory(socketPath);
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : "connection failed";
|
||||
failStatus(`Unable to query dictation service: ${detail}`);
|
||||
return;
|
||||
}
|
||||
statusSocket = current;
|
||||
current.onConnect(() => {
|
||||
if (statusSocket !== current || !statusRequestId) return;
|
||||
sendStatus(current, {
|
||||
version: PROTOCOL_VERSION,
|
||||
type: "hello",
|
||||
requestId: statusRequestId,
|
||||
});
|
||||
});
|
||||
current.onData((chunk) => consumeStatusData(chunk, current));
|
||||
current.onError((error) => {
|
||||
if (statusSocket !== current) return;
|
||||
failStatus(`Dictation status connection failed: ${error.message}`);
|
||||
});
|
||||
current.onClose(() => {
|
||||
if (statusSocket !== current) return;
|
||||
failStatus("Dictation status service disconnected.");
|
||||
});
|
||||
};
|
||||
|
||||
const startDictation = (ctx: ExtensionContext) => {
|
||||
if (state !== "idle") return;
|
||||
closeStatusQuery(true);
|
||||
state = "connecting";
|
||||
activeContext = ctx;
|
||||
activeGenerationId = newId("generation");
|
||||
@@ -438,18 +642,23 @@ export function createDictateExtension(options: DictateExtensionOptions = {}) {
|
||||
};
|
||||
|
||||
pi.registerShortcut("alt+m", {
|
||||
description: "Start or stop local voice dictation",
|
||||
description: "Start or stop voice dictation",
|
||||
handler: async (ctx) => toggleDictation(ctx),
|
||||
});
|
||||
pi.registerShortcut("alt+n", {
|
||||
description: "Cancel local voice dictation",
|
||||
description: "Cancel voice dictation",
|
||||
handler: async () => cancelDictation(),
|
||||
});
|
||||
pi.registerCommand("dictate-cancel", {
|
||||
description: "Cancel local voice dictation",
|
||||
description: "Cancel voice dictation",
|
||||
handler: async () => cancelDictation(),
|
||||
});
|
||||
pi.registerCommand("dictate-status", {
|
||||
description: "Show the selected dictation backend",
|
||||
handler: async (_args, ctx) => statusDictation(ctx),
|
||||
});
|
||||
pi.on("session_shutdown", async () => {
|
||||
closeStatusQuery(true);
|
||||
if (state === "idle") return;
|
||||
if (negotiated) sendGenerationCommand("cancel");
|
||||
reset();
|
||||
|
||||
Reference in New Issue
Block a user