diff --git a/packages/app/e2e/new-workspace-dictation-submit.spec.ts b/packages/app/e2e/new-workspace-dictation-submit.spec.ts new file mode 100644 index 000000000..b96f97dd0 --- /dev/null +++ b/packages/app/e2e/new-workspace-dictation-submit.spec.ts @@ -0,0 +1,195 @@ +import { expect, test, type Page } from "./fixtures"; +import { gotoAppShell } from "./helpers/app"; +import { daemonWsRoutePattern } from "./helpers/daemon-port"; +import { openNewWorkspaceComposer, selectWorkspaceIsolation } from "./helpers/new-workspace"; +import { seedWorkspace } from "./helpers/seed-client"; +import { waitForSidebarHydration } from "./helpers/workspace-ui"; + +type WebSocketMessage = string | Buffer; + +const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace"; +const CREATE_FAILURE = "Synthetic workspace creation failure"; + +function parseEnvelope(message: WebSocketMessage): { + type?: unknown; + message?: Record; +} | null { + const raw = typeof message === "string" ? message : message.toString("utf8"); + try { + return JSON.parse(raw) as { type?: unknown; message?: Record }; + } catch { + return null; + } +} + +function sessionMessage(message: WebSocketMessage): Record | null { + const envelope = parseEnvelope(message); + return envelope?.type === "session" && envelope.message ? envelope.message : null; +} + +function sendSessionMessage( + ws: { send(message: string): void }, + message: Record, +): void { + ws.send(JSON.stringify({ type: "session", message })); +} + +function enableDictationCapability(message: WebSocketMessage): WebSocketMessage { + const envelope = parseEnvelope(message); + const payload = envelope?.message?.payload; + if ( + envelope?.message?.type !== "status" || + !payload || + typeof payload !== "object" || + (payload as { status?: unknown }).status !== "server_info" + ) { + return message; + } + + (payload as Record).capabilities = { + voice: { + dictation: { enabled: true, reason: "" }, + voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, + }, + }; + return JSON.stringify(envelope); +} + +async function installSyntheticMicrophone(page: Page): Promise { + await page.addInitScript(() => { + const mediaDevices = navigator.mediaDevices; + Object.defineProperty(mediaDevices, "getUserMedia", { + configurable: true, + value: async () => { + const context = new AudioContext(); + const oscillator = context.createOscillator(); + const destination = context.createMediaStreamDestination(); + oscillator.connect(destination); + oscillator.start(); + return destination.stream; + }, + }); + }); +} + +async function installDictationFailureHarness(page: Page) { + let resolveAudioChunk!: () => void; + let resolveCreateRequest!: () => void; + const audioChunk = new Promise((resolve) => { + resolveAudioChunk = resolve; + }); + const createRequest = new Promise((resolve) => { + resolveCreateRequest = resolve; + }); + let failCreate: (() => void) | null = null; + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + + ws.onMessage((message) => { + const request = sessionMessage(message); + const type = request?.type; + const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null; + + if (type === "dictation_stream_start" && dictationId) { + sendSessionMessage(ws, { + type: "dictation_stream_ack", + payload: { dictationId, ackSeq: -1 }, + }); + return; + } + if (type === "dictation_stream_chunk" && dictationId) { + const seq = typeof request?.seq === "number" ? request.seq : 0; + sendSessionMessage(ws, { + type: "dictation_stream_ack", + payload: { dictationId, ackSeq: seq }, + }); + resolveAudioChunk(); + return; + } + if (type === "dictation_stream_finish" && dictationId) { + sendSessionMessage(ws, { + type: "dictation_stream_finish_accepted", + payload: { dictationId, timeoutMs: 5_000 }, + }); + sendSessionMessage(ws, { + type: "dictation_stream_final", + payload: { dictationId, text: TRANSCRIPT }, + }); + return; + } + if (request && type === "workspace.create.request" && typeof request.requestId === "string") { + const requestId = request.requestId; + failCreate = () => { + sendSessionMessage(ws, { + type: "workspace.create.response", + payload: { + requestId, + workspace: null, + setupTerminalId: null, + error: CREATE_FAILURE, + }, + }); + }; + resolveCreateRequest(); + return; + } + + server.send(message); + }); + + server.onMessage((message) => ws.send(enableDictationCapability(message))); + }); + + return { + waitForAudio: () => audioChunk, + waitForCreateRequest: () => createRequest, + failWorkspaceCreation: () => { + if (!failCreate) { + throw new Error("Workspace creation request has not been received"); + } + failCreate(); + }, + }; +} + +async function dictateAndSend(page: Page, waitForAudio: () => Promise): Promise { + await page.getByRole("button", { name: "Start dictation" }).click(); + await waitForAudio(); + await page.getByRole("button", { name: "Insert transcription and send" }).click(); +} + +test.describe("New Workspace dictation submit", () => { + test.describe.configure({ timeout: 240_000 }); + + test("keeps the spoken prompt visible while pending and after failure", async ({ page }) => { + const seeded = await seedWorkspace({ repoPrefix: "dictation-submit-" }); + await installSyntheticMicrophone(page); + const harness = await installDictationFailureHarness(page); + + try { + await gotoAppShell(page); + await waitForSidebarHydration(page); + await openNewWorkspaceComposer(page, { + projectKey: seeded.projectKey, + projectDisplayName: seeded.projectDisplayName, + }); + await selectWorkspaceIsolation(page, "local"); + + await dictateAndSend(page, harness.waitForAudio); + await harness.waitForCreateRequest(); + + const composer = page.getByRole("textbox", { name: "Message agent..." }); + await expect(composer).toHaveValue(TRANSCRIPT); + await expect(composer).not.toBeEditable(); + + harness.failWorkspaceCreation(); + + await expect(composer).toBeEditable(); + await expect(composer).toHaveValue(TRANSCRIPT); + await expect(page.getByText(CREATE_FAILURE).first()).toBeVisible(); + } finally { + await seeded.cleanup(); + } + }); +}); diff --git a/packages/app/src/composer/input/input.tsx b/packages/app/src/composer/input/input.tsx index cbe92e330..a624af981 100644 --- a/packages/app/src/composer/input/input.tsx +++ b/packages/app/src/composer/input/input.tsx @@ -66,6 +66,7 @@ import { resolveVoiceTooltipText, } from "./labels"; import { + applyDictationTranscript, computeCanStartDictation, resolveComposerSurfacePresentation, runAlternateSendAction, @@ -778,42 +779,6 @@ function SendButtonTooltip({ ); } -interface DictationTranscriptContext { - value: string; - defaultSendBehavior: "interrupt" | "queue"; - isAgentRunning: boolean; - onQueue: ((payload: MessagePayload) => void) | undefined; - onSubmit: (payload: MessagePayload) => void; - onChangeText: (text: string) => void; - attachments: ComposerAttachment[]; - cwd: string; - autoSend: boolean; -} - -function applyDictationTranscript(text: string, ctx: DictationTranscriptContext): void { - if (!text) return; - const shouldPad = ctx.value.length > 0 && !/\s$/.test(ctx.value); - const nextValue = `${ctx.value}${shouldPad ? " " : ""}${text}`; - - if (!ctx.autoSend) { - ctx.onChangeText(nextValue); - return; - } - - if (ctx.defaultSendBehavior === "queue" && ctx.isAgentRunning && ctx.onQueue) { - ctx.onQueue({ text: nextValue, attachments: ctx.attachments, cwd: ctx.cwd }); - ctx.onChangeText(""); - return; - } - - ctx.onSubmit({ - text: nextValue, - attachments: ctx.attachments, - cwd: ctx.cwd, - forceSend: ctx.isAgentRunning || undefined, - }); -} - interface ToggleRealtimeVoiceContext { voice: | { diff --git a/packages/app/src/composer/input/state.test.ts b/packages/app/src/composer/input/state.test.ts index 90232fb2e..d6265b063 100644 --- a/packages/app/src/composer/input/state.test.ts +++ b/packages/app/src/composer/input/state.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { + applyDictationTranscript, computeCanStartDictation, resolveComposerSurfacePresentation, runAlternateSendAction, @@ -163,6 +164,29 @@ describe("dictation keyboard behavior", () => { }); }); +describe("dictation transcript behavior", () => { + it("publishes an auto-sent transcript to the composer before submitting it", () => { + const actions: string[] = []; + + applyDictationTranscript("spoken prompt", { + value: "typed context", + defaultSendBehavior: "interrupt", + isAgentRunning: false, + onQueue: undefined, + onChangeText: (text) => actions.push(`change:${text}`), + onSubmit: (payload) => actions.push(`submit:${payload.text}`), + attachments: [], + cwd: "/repo", + autoSend: true, + }); + + expect(actions).toEqual([ + "change:typed context spoken prompt", + "submit:typed context spoken prompt", + ]); + }); +}); + describe("composer send behavior", () => { function actions() { const calls: string[] = []; diff --git a/packages/app/src/composer/input/state.ts b/packages/app/src/composer/input/state.ts index 6d3888679..733a30cc4 100644 --- a/packages/app/src/composer/input/state.ts +++ b/packages/app/src/composer/input/state.ts @@ -46,6 +46,44 @@ interface SendActionContext { handleQueueMessage: () => void; } +interface DictationTranscriptContext { + value: string; + defaultSendBehavior: SendBehavior; + isAgentRunning: boolean; + onQueue: ((payload: MessagePayload) => void) | undefined; + onSubmit: (payload: MessagePayload) => void; + onChangeText: (text: string) => void; + attachments: MessagePayload["attachments"]; + cwd: string; + autoSend: boolean; +} + +export function applyDictationTranscript(text: string, ctx: DictationTranscriptContext): void { + if (!text) return; + const shouldPad = ctx.value.length > 0 && !/\s$/.test(ctx.value); + const nextValue = `${ctx.value}${shouldPad ? " " : ""}${text}`; + + if (!ctx.autoSend) { + ctx.onChangeText(nextValue); + return; + } + + ctx.onChangeText(nextValue); + + if (ctx.defaultSendBehavior === "queue" && ctx.isAgentRunning && ctx.onQueue) { + ctx.onQueue({ text: nextValue, attachments: ctx.attachments, cwd: ctx.cwd }); + ctx.onChangeText(""); + return; + } + + ctx.onSubmit({ + text: nextValue, + attachments: ctx.attachments, + cwd: ctx.cwd, + forceSend: ctx.isAgentRunning || undefined, + }); +} + interface MessageInputKeyboardActions { focusInput: () => void; isDictationRecording: () => boolean;