From a059af1edac759395a6b5b4b76471e73cacba186 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sat, 1 Aug 2026 12:28:55 +0200 Subject: [PATCH] fix(app): preserve dictation text during submit (#2745) Dictation auto-send bypassed the controlled composer update, so preserved draft surfaces could lock while showing an empty prompt. Publish the transcript before invoking the existing submit lifecycle so pending and failed submissions retain the user's spoken input. --- .../new-workspace-dictation-submit.spec.ts | 195 ++++++++++++++++++ packages/app/src/composer/input/input.tsx | 37 +--- packages/app/src/composer/input/state.test.ts | 24 +++ packages/app/src/composer/input/state.ts | 38 ++++ 4 files changed, 258 insertions(+), 36 deletions(-) create mode 100644 packages/app/e2e/new-workspace-dictation-submit.spec.ts diff --git a/packages/app/e2e/new-workspace-dictation-submit.spec.ts b/packages/app/e2e/new-workspace-dictation-submit.spec.ts new file mode 100644 index 000000000..b96f97dd0 --- /dev/null +++ b/packages/app/e2e/new-workspace-dictation-submit.spec.ts @@ -0,0 +1,195 @@ +import { expect, test, type Page } from "./fixtures"; +import { gotoAppShell } from "./helpers/app"; +import { daemonWsRoutePattern } from "./helpers/daemon-port"; +import { openNewWorkspaceComposer, selectWorkspaceIsolation } from "./helpers/new-workspace"; +import { seedWorkspace } from "./helpers/seed-client"; +import { waitForSidebarHydration } from "./helpers/workspace-ui"; + +type WebSocketMessage = string | Buffer; + +const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace"; +const CREATE_FAILURE = "Synthetic workspace creation failure"; + +function parseEnvelope(message: WebSocketMessage): { + type?: unknown; + message?: Record; +} | null { + const raw = typeof message === "string" ? message : message.toString("utf8"); + try { + return JSON.parse(raw) as { type?: unknown; message?: Record }; + } catch { + return null; + } +} + +function sessionMessage(message: WebSocketMessage): Record | null { + const envelope = parseEnvelope(message); + return envelope?.type === "session" && envelope.message ? envelope.message : null; +} + +function sendSessionMessage( + ws: { send(message: string): void }, + message: Record, +): void { + ws.send(JSON.stringify({ type: "session", message })); +} + +function enableDictationCapability(message: WebSocketMessage): WebSocketMessage { + const envelope = parseEnvelope(message); + const payload = envelope?.message?.payload; + if ( + envelope?.message?.type !== "status" || + !payload || + typeof payload !== "object" || + (payload as { status?: unknown }).status !== "server_info" + ) { + return message; + } + + (payload as Record).capabilities = { + voice: { + dictation: { enabled: true, reason: "" }, + voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, + }, + }; + return JSON.stringify(envelope); +} + +async function installSyntheticMicrophone(page: Page): Promise { + await page.addInitScript(() => { + const mediaDevices = navigator.mediaDevices; + Object.defineProperty(mediaDevices, "getUserMedia", { + configurable: true, + value: async () => { + const context = new AudioContext(); + const oscillator = context.createOscillator(); + const destination = context.createMediaStreamDestination(); + oscillator.connect(destination); + oscillator.start(); + return destination.stream; + }, + }); + }); +} + +async function installDictationFailureHarness(page: Page) { + let resolveAudioChunk!: () => void; + let resolveCreateRequest!: () => void; + const audioChunk = new Promise((resolve) => { + resolveAudioChunk = resolve; + }); + const createRequest = new Promise((resolve) => { + resolveCreateRequest = resolve; + }); + let failCreate: (() => void) | null = null; + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + + ws.onMessage((message) => { + const request = sessionMessage(message); + const type = request?.type; + const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null; + + if (type === "dictation_stream_start" && dictationId) { + sendSessionMessage(ws, { + type: "dictation_stream_ack", + payload: { dictationId, ackSeq: -1 }, + }); + return; + } + if (type === "dictation_stream_chunk" && dictationId) { + const seq = typeof request?.seq === "number" ? request.seq : 0; + sendSessionMessage(ws, { + type: "dictation_stream_ack", + payload: { dictationId, ackSeq: seq }, + }); + resolveAudioChunk(); + return; + } + if (type === "dictation_stream_finish" && dictationId) { + sendSessionMessage(ws, { + type: "dictation_stream_finish_accepted", + payload: { dictationId, timeoutMs: 5_000 }, + }); + sendSessionMessage(ws, { + type: "dictation_stream_final", + payload: { dictationId, text: TRANSCRIPT }, + }); + return; + } + if (request && type === "workspace.create.request" && typeof request.requestId === "string") { + const requestId = request.requestId; + failCreate = () => { + sendSessionMessage(ws, { + type: "workspace.create.response", + payload: { + requestId, + workspace: null, + setupTerminalId: null, + error: CREATE_FAILURE, + }, + }); + }; + resolveCreateRequest(); + return; + } + + server.send(message); + }); + + server.onMessage((message) => ws.send(enableDictationCapability(message))); + }); + + return { + waitForAudio: () => audioChunk, + waitForCreateRequest: () => createRequest, + failWorkspaceCreation: () => { + if (!failCreate) { + throw new Error("Workspace creation request has not been received"); + } + failCreate(); + }, + }; +} + +async function dictateAndSend(page: Page, waitForAudio: () => Promise): Promise { + await page.getByRole("button", { name: "Start dictation" }).click(); + await waitForAudio(); + await page.getByRole("button", { name: "Insert transcription and send" }).click(); +} + +test.describe("New Workspace dictation submit", () => { + test.describe.configure({ timeout: 240_000 }); + + test("keeps the spoken prompt visible while pending and after failure", async ({ page }) => { + const seeded = await seedWorkspace({ repoPrefix: "dictation-submit-" }); + await installSyntheticMicrophone(page); + const harness = await installDictationFailureHarness(page); + + try { + await gotoAppShell(page); + await waitForSidebarHydration(page); + await openNewWorkspaceComposer(page, { + projectKey: seeded.projectKey, + projectDisplayName: seeded.projectDisplayName, + }); + await selectWorkspaceIsolation(page, "local"); + + await dictateAndSend(page, harness.waitForAudio); + await harness.waitForCreateRequest(); + + const composer = page.getByRole("textbox", { name: "Message agent..." }); + await expect(composer).toHaveValue(TRANSCRIPT); + await expect(composer).not.toBeEditable(); + + harness.failWorkspaceCreation(); + + await expect(composer).toBeEditable(); + await expect(composer).toHaveValue(TRANSCRIPT); + await expect(page.getByText(CREATE_FAILURE).first()).toBeVisible(); + } finally { + await seeded.cleanup(); + } + }); +}); diff --git a/packages/app/src/composer/input/input.tsx b/packages/app/src/composer/input/input.tsx index cbe92e330..a624af981 100644 --- a/packages/app/src/composer/input/input.tsx +++ b/packages/app/src/composer/input/input.tsx @@ -66,6 +66,7 @@ import { resolveVoiceTooltipText, } from "./labels"; import { + applyDictationTranscript, computeCanStartDictation, resolveComposerSurfacePresentation, runAlternateSendAction, @@ -778,42 +779,6 @@ function SendButtonTooltip({ ); } -interface DictationTranscriptContext { - value: string; - defaultSendBehavior: "interrupt" | "queue"; - isAgentRunning: boolean; - onQueue: ((payload: MessagePayload) => void) | undefined; - onSubmit: (payload: MessagePayload) => void; - onChangeText: (text: string) => void; - attachments: ComposerAttachment[]; - cwd: string; - autoSend: boolean; -} - -function applyDictationTranscript(text: string, ctx: DictationTranscriptContext): void { - if (!text) return; - const shouldPad = ctx.value.length > 0 && !/\s$/.test(ctx.value); - const nextValue = `${ctx.value}${shouldPad ? " " : ""}${text}`; - - if (!ctx.autoSend) { - ctx.onChangeText(nextValue); - return; - } - - if (ctx.defaultSendBehavior === "queue" && ctx.isAgentRunning && ctx.onQueue) { - ctx.onQueue({ text: nextValue, attachments: ctx.attachments, cwd: ctx.cwd }); - ctx.onChangeText(""); - return; - } - - ctx.onSubmit({ - text: nextValue, - attachments: ctx.attachments, - cwd: ctx.cwd, - forceSend: ctx.isAgentRunning || undefined, - }); -} - interface ToggleRealtimeVoiceContext { voice: | { diff --git a/packages/app/src/composer/input/state.test.ts b/packages/app/src/composer/input/state.test.ts index 90232fb2e..d6265b063 100644 --- a/packages/app/src/composer/input/state.test.ts +++ b/packages/app/src/composer/input/state.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { + applyDictationTranscript, computeCanStartDictation, resolveComposerSurfacePresentation, runAlternateSendAction, @@ -163,6 +164,29 @@ describe("dictation keyboard behavior", () => { }); }); +describe("dictation transcript behavior", () => { + it("publishes an auto-sent transcript to the composer before submitting it", () => { + const actions: string[] = []; + + applyDictationTranscript("spoken prompt", { + value: "typed context", + defaultSendBehavior: "interrupt", + isAgentRunning: false, + onQueue: undefined, + onChangeText: (text) => actions.push(`change:${text}`), + onSubmit: (payload) => actions.push(`submit:${payload.text}`), + attachments: [], + cwd: "/repo", + autoSend: true, + }); + + expect(actions).toEqual([ + "change:typed context spoken prompt", + "submit:typed context spoken prompt", + ]); + }); +}); + describe("composer send behavior", () => { function actions() { const calls: string[] = []; diff --git a/packages/app/src/composer/input/state.ts b/packages/app/src/composer/input/state.ts index 6d3888679..733a30cc4 100644 --- a/packages/app/src/composer/input/state.ts +++ b/packages/app/src/composer/input/state.ts @@ -46,6 +46,44 @@ interface SendActionContext { handleQueueMessage: () => void; } +interface DictationTranscriptContext { + value: string; + defaultSendBehavior: SendBehavior; + isAgentRunning: boolean; + onQueue: ((payload: MessagePayload) => void) | undefined; + onSubmit: (payload: MessagePayload) => void; + onChangeText: (text: string) => void; + attachments: MessagePayload["attachments"]; + cwd: string; + autoSend: boolean; +} + +export function applyDictationTranscript(text: string, ctx: DictationTranscriptContext): void { + if (!text) return; + const shouldPad = ctx.value.length > 0 && !/\s$/.test(ctx.value); + const nextValue = `${ctx.value}${shouldPad ? " " : ""}${text}`; + + if (!ctx.autoSend) { + ctx.onChangeText(nextValue); + return; + } + + ctx.onChangeText(nextValue); + + if (ctx.defaultSendBehavior === "queue" && ctx.isAgentRunning && ctx.onQueue) { + ctx.onQueue({ text: nextValue, attachments: ctx.attachments, cwd: ctx.cwd }); + ctx.onChangeText(""); + return; + } + + ctx.onSubmit({ + text: nextValue, + attachments: ctx.attachments, + cwd: ctx.cwd, + forceSend: ctx.isAgentRunning || undefined, + }); +} + interface MessageInputKeyboardActions { focusInput: () => void; isDictationRecording: () => boolean;