From adc0c017826f5f2dfe87577e47240ffab028ec8e Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Thu, 2 Jul 2026 18:43:54 +0200 Subject: [PATCH] Fail fast on unpainted browser screenshots --- .../browser-automation/service.test.ts | 141 +++++++++++++++++- .../features/browser-automation/service.ts | 83 +++++++++-- .../browser-automation/rpc-schemas.test.ts | 24 +++ .../src/browser-automation/rpc-schemas.ts | 1 + .../src/server/browser-tools/tools.test.ts | 25 ++++ .../server/src/server/browser-tools/tools.ts | 2 + 6 files changed, 265 insertions(+), 11 deletions(-) diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index b53062da5..f35ab5fd7 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -1,6 +1,6 @@ import { resolve as resolvePath } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { BrowserSnapshotEngine } from "./snapshot-engine.js"; import type { TabContents, BrowserRegistry } from "./service.js"; import { executeAutomationCommand } from "./service.js"; @@ -1360,6 +1360,53 @@ describe("executeAutomationCommand", () => { }); }); + it("returns screenshot_no_frame when full-page CDP capture never paints", async () => { + vi.useFakeTimers(); + try { + const tab = fakeTab({ + id: 22, + sendDebugCommand: async (command) => { + if (command === "Page.getLayoutMetrics") { + return { contentSize: { width: 390, height: 1200 } }; + } + return new Promise(() => {}); + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => + workspaceId === "workspace-a" ? "a" : null, + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const resultPromise = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-full-page-no-frame", + workspaceId: "workspace-a", + command: { command: "full_page_screenshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + await vi.advanceTimersByTimeAsync(5_000); + + await expect(resultPromise).resolves.toEqual({ + requestId: "r-full-page-no-frame", + ok: false, + error: { + code: "screenshot_no_frame", + message: + "The browser tab has no painted frame. Focus the tab in the app, then try again.", + retryable: false, + }, + }); + } finally { + vi.useRealTimers(); + } + }); + it("exports the target tab as PDF", async () => { const printOptions: Record[] = []; const tab = fakeTab({ @@ -1933,6 +1980,56 @@ describe("executeAutomationCommand", () => { expect(captureParams).toEqual({ format: "png", fromSurface: false }); }); + it("returns screenshot_no_frame when CDP viewport capture never paints", async () => { + vi.useFakeTimers(); + try { + const tab = fakeTab({ + id: 13, + sendDebugCommand: async (command) => { + if (command === "Page.captureScreenshot") { + return new Promise(() => {}); + } + throw new Error(`Unexpected CDP command ${command}`); + }, + capturePage: async () => { + throw new Error("capturePage should not be used when CDP is available"); + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => + workspaceId === "workspace-a" ? "a" : null, + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const resultPromise = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-screenshot-cdp-no-frame", + workspaceId: "workspace-a", + command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + await vi.advanceTimersByTimeAsync(5_000); + + await expect(resultPromise).resolves.toEqual({ + requestId: "r-screenshot-cdp-no-frame", + ok: false, + error: { + code: "screenshot_no_frame", + message: + "The browser tab has no painted frame. Focus the tab in the app, then try again.", + retryable: false, + }, + }); + } finally { + vi.useRealTimers(); + } + }); + it("captures a PNG screenshot from the active browser", async () => { const tab = fakeTab({ id: 13, @@ -1971,6 +2068,48 @@ describe("executeAutomationCommand", () => { }, }); }); + + it("returns screenshot_no_frame when capturePage never paints", async () => { + vi.useFakeTimers(); + try { + const tab = fakeTab({ + id: 13, + capturePage: async () => new Promise(() => {}), + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => + workspaceId === "workspace-a" ? "a" : null, + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const resultPromise = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-screenshot-no-frame", + workspaceId: "workspace-a", + command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + await vi.advanceTimersByTimeAsync(5_000); + + await expect(resultPromise).resolves.toEqual({ + requestId: "r-screenshot-no-frame", + ok: false, + error: { + code: "screenshot_no_frame", + message: + "The browser tab has no painted frame. Focus the tab in the app, then try again.", + retryable: false, + }, + }); + } finally { + vi.useRealTimers(); + } + }); }); describe("unsupported command", () => { diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index 38d2a8b74..780a4b864 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -58,6 +58,9 @@ type FailurePayload = Extract; const defaultSnapshotEngine = new BrowserSnapshotEngine(); const DEFAULT_WAIT_TIMEOUT_MS = 5_000; const WAIT_POLL_INTERVAL_MS = 25; +const PIXEL_CAPTURE_TIMEOUT_MS = 5_000; +const SCREENSHOT_NO_FRAME_MESSAGE = + "The browser tab has no painted frame. Focus the tab in the app, then try again."; const ALLOWED_PAGE_URL_PROTOCOLS = new Set(["http:", "https:"]); function fail( @@ -69,6 +72,38 @@ function fail( return { requestId, ok: false, error: { code, message, retryable } }; } +class ScreenshotNoFrameError extends Error { + public constructor() { + super(SCREENSHOT_NO_FRAME_MESSAGE); + this.name = "ScreenshotNoFrameError"; + } +} + +function isScreenshotNoFrameError(error: unknown): error is ScreenshotNoFrameError { + return error instanceof ScreenshotNoFrameError; +} + +function screenshotNoFrameFailure(requestId: string): FailurePayload { + return fail(requestId, "screenshot_no_frame", SCREENSHOT_NO_FRAME_MESSAGE); +} + +async function withPixelCaptureTimeout(capture: Promise): Promise { + let timeoutId: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject(new ScreenshotNoFrameError()); + }, PIXEL_CAPTURE_TIMEOUT_MS); + }); + + try { + return await Promise.race([capture, timeout]); + } finally { + if (timeoutId) { + clearTimeout(timeoutId); + } + } +} + function tabInfoFromContents( browserId: string, contents: TabContents, @@ -1070,10 +1105,20 @@ async function executeScreenshot( return target; } if (target.contents.sendDebugCommand) { - const screenshot = (await target.contents.sendDebugCommand("Page.captureScreenshot", { - format: "png", - fromSurface: false, - })) as CdpCaptureScreenshotResult; + let screenshot: CdpCaptureScreenshotResult; + try { + screenshot = (await withPixelCaptureTimeout( + target.contents.sendDebugCommand("Page.captureScreenshot", { + format: "png", + fromSurface: false, + }), + )) as CdpCaptureScreenshotResult; + } catch (error) { + if (isScreenshotNoFrameError(error)) { + return screenshotNoFrameFailure(requestId); + } + throw error; + } if (!screenshot.data) { return fail(requestId, "browser_unsupported", "browser_screenshot returned no data"); } @@ -1091,7 +1136,15 @@ async function executeScreenshot( }, }; } - const image = await target.contents.capturePage(); + let image: TabImage; + try { + image = await withPixelCaptureTimeout(target.contents.capturePage()); + } catch (error) { + if (isScreenshotNoFrameError(error)) { + return screenshotNoFrameFailure(requestId); + } + throw error; + } const size = image.getSize(); return { requestId, @@ -1180,11 +1233,21 @@ async function executeFullPageScreenshot( const metrics = await getCdpLayoutMetrics(target.contents); const width = metrics.contentWidth; const height = metrics.contentHeight; - const screenshot = (await target.contents.sendDebugCommand("Page.captureScreenshot", { - format: "png", - captureBeyondViewport: true, - clip: { x: 0, y: 0, width, height, scale: 1 }, - })) as CdpCaptureScreenshotResult; + let screenshot: CdpCaptureScreenshotResult; + try { + screenshot = (await withPixelCaptureTimeout( + target.contents.sendDebugCommand("Page.captureScreenshot", { + format: "png", + captureBeyondViewport: true, + clip: { x: 0, y: 0, width, height, scale: 1 }, + }), + )) as CdpCaptureScreenshotResult; + } catch (error) { + if (isScreenshotNoFrameError(error)) { + return screenshotNoFrameFailure(requestId); + } + throw error; + } if (!screenshot.data) { return fail(requestId, "browser_unsupported", "browser_full_page_screenshot returned no data"); } diff --git a/packages/protocol/src/browser-automation/rpc-schemas.test.ts b/packages/protocol/src/browser-automation/rpc-schemas.test.ts index 52dba84aa..eba3bad50 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.test.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.test.ts @@ -374,6 +374,30 @@ describe("browser automation execute RPC schemas", () => { height: 50, }, }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-shot-no-frame", + ok: false, + error: { + code: "screenshot_no_frame", + message: + "The browser tab has no painted frame. Focus the tab in the app, then try again.", + retryable: false, + }, + }, + }).payload, + ).toEqual({ + requestId: "req-shot-no-frame", + ok: false, + error: { + code: "screenshot_no_frame", + message: "The browser tab has no painted frame. Focus the tab in the app, then try again.", + retryable: false, + }, + }); }); test("parses form control commands and responses", () => { diff --git a/packages/protocol/src/browser-automation/rpc-schemas.ts b/packages/protocol/src/browser-automation/rpc-schemas.ts index 4d741e8e6..62125515a 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.ts @@ -7,6 +7,7 @@ export const BrowserAutomationErrorCodeSchema = z.enum([ "browser_tab_not_found", "browser_tab_closed", "browser_timeout", + "screenshot_no_frame", "browser_denied", "browser_unsupported", "browser_stale_ref", diff --git a/packages/server/src/server/browser-tools/tools.test.ts b/packages/server/src/server/browser-tools/tools.test.ts index 88a88a90d..67eb3c8d5 100644 --- a/packages/server/src/server/browser-tools/tools.test.ts +++ b/packages/server/src/server/browser-tools/tools.test.ts @@ -908,4 +908,29 @@ describe("registerBrowserTools", () => { retryable: true, }); }); + + it("preserves screenshot no-frame broker messages", async () => { + const message = + "The browser tab has no painted frame. Focus the tab in the app, then try again."; + const harness = createHarness({ + brokerResponse: { + requestId: "req-no-frame", + ok: false, + error: { + code: "screenshot_no_frame", + message, + retryable: false, + }, + }, + }); + + const response = await tool(harness, "browser_screenshot").handler({}); + + expect(response.content).toEqual([{ type: "text", text: message }]); + expect(response.structuredContent?.error).toEqual({ + code: "screenshot_no_frame", + message, + retryable: false, + }); + }); }); diff --git a/packages/server/src/server/browser-tools/tools.ts b/packages/server/src/server/browser-tools/tools.ts index eed729a63..3d8236cb9 100644 --- a/packages/server/src/server/browser-tools/tools.ts +++ b/packages/server/src/server/browser-tools/tools.ts @@ -1116,6 +1116,8 @@ function summarizeBrowserError( return "No active browser tab is available. Call browser_new_tab to create one, then omit browserId for the active tab or use the returned browserId."; case "browser_timeout": return "The browser did not respond before the timeout. Try again or check the desktop app."; + case "screenshot_no_frame": + return error.message; case "browser_unsupported": return "This desktop build does not support that browser automation request yet."; case "browser_stale_ref":