Fail fast on unpainted browser screenshots

packages/desktop/src/features/browser-automation/service.test.ts

@@ -1,6 +1,6 @@
 import { resolve as resolvePath } from "node:path";
 
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import { BrowserSnapshotEngine } from "./snapshot-engine.js";
 import type { TabContents, BrowserRegistry } from "./service.js";
 import { executeAutomationCommand } from "./service.js";
@@ -1360,6 +1360,53 @@ describe("executeAutomationCommand", () => {
       });
     });
 
+    it("returns screenshot_no_frame when full-page CDP capture never paints", async () => {
+      vi.useFakeTimers();
+      try {
+        const tab = fakeTab({
+          id: 22,
+          sendDebugCommand: async (command) => {
+            if (command === "Page.getLayoutMetrics") {
+              return { contentSize: { width: 390, height: 1200 } };
+            }
+            return new Promise<never>(() => {});
+          },
+        });
+        const registry = createRegistry({
+          getWorkspaceActiveTabContents: (workspaceId) =>
+            workspaceId === "workspace-a" ? tab : null,
+          getWorkspaceActiveBrowserId: (workspaceId) =>
+            workspaceId === "workspace-a" ? "a" : null,
+          getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null),
+        });
+
+        const resultPromise = executeAutomationCommand(
+          {
+            type: "browser.automation.execute.request",
+            requestId: "r-full-page-no-frame",
+            workspaceId: "workspace-a",
+            command: { command: "full_page_screenshot", args: { workspaceId: "workspace-a" } },
+          },
+          registry,
+        );
+
+        await vi.advanceTimersByTimeAsync(5_000);
+
+        await expect(resultPromise).resolves.toEqual({
+          requestId: "r-full-page-no-frame",
+          ok: false,
+          error: {
+            code: "screenshot_no_frame",
+            message:
+              "The browser tab has no painted frame. Focus the tab in the app, then try again.",
+            retryable: false,
+          },
+        });
+      } finally {
+        vi.useRealTimers();
+      }
+    });
+
     it("exports the target tab as PDF", async () => {
       const printOptions: Record<string, unknown>[] = [];
       const tab = fakeTab({
@@ -1933,6 +1980,56 @@ describe("executeAutomationCommand", () => {
       expect(captureParams).toEqual({ format: "png", fromSurface: false });
     });
 
+    it("returns screenshot_no_frame when CDP viewport capture never paints", async () => {
+      vi.useFakeTimers();
+      try {
+        const tab = fakeTab({
+          id: 13,
+          sendDebugCommand: async (command) => {
+            if (command === "Page.captureScreenshot") {
+              return new Promise<never>(() => {});
+            }
+            throw new Error(`Unexpected CDP command ${command}`);
+          },
+          capturePage: async () => {
+            throw new Error("capturePage should not be used when CDP is available");
+          },
+        });
+        const registry = createRegistry({
+          getWorkspaceActiveTabContents: (workspaceId) =>
+            workspaceId === "workspace-a" ? tab : null,
+          getWorkspaceActiveBrowserId: (workspaceId) =>
+            workspaceId === "workspace-a" ? "a" : null,
+          getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null),
+        });
+
+        const resultPromise = executeAutomationCommand(
+          {
+            type: "browser.automation.execute.request",
+            requestId: "r-screenshot-cdp-no-frame",
+            workspaceId: "workspace-a",
+            command: { command: "screenshot", args: { workspaceId: "workspace-a" } },
+          },
+          registry,
+        );
+
+        await vi.advanceTimersByTimeAsync(5_000);
+
+        await expect(resultPromise).resolves.toEqual({
+          requestId: "r-screenshot-cdp-no-frame",
+          ok: false,
+          error: {
+            code: "screenshot_no_frame",
+            message:
+              "The browser tab has no painted frame. Focus the tab in the app, then try again.",
+            retryable: false,
+          },
+        });
+      } finally {
+        vi.useRealTimers();
+      }
+    });
+
     it("captures a PNG screenshot from the active browser", async () => {
       const tab = fakeTab({
         id: 13,
@@ -1971,6 +2068,48 @@ describe("executeAutomationCommand", () => {
         },
       });
     });
+
+    it("returns screenshot_no_frame when capturePage never paints", async () => {
+      vi.useFakeTimers();
+      try {
+        const tab = fakeTab({
+          id: 13,
+          capturePage: async () => new Promise<never>(() => {}),
+        });
+        const registry = createRegistry({
+          getWorkspaceActiveTabContents: (workspaceId) =>
+            workspaceId === "workspace-a" ? tab : null,
+          getWorkspaceActiveBrowserId: (workspaceId) =>
+            workspaceId === "workspace-a" ? "a" : null,
+          getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null),
+        });
+
+        const resultPromise = executeAutomationCommand(
+          {
+            type: "browser.automation.execute.request",
+            requestId: "r-screenshot-no-frame",
+            workspaceId: "workspace-a",
+            command: { command: "screenshot", args: { workspaceId: "workspace-a" } },
+          },
+          registry,
+        );
+
+        await vi.advanceTimersByTimeAsync(5_000);
+
+        await expect(resultPromise).resolves.toEqual({
+          requestId: "r-screenshot-no-frame",
+          ok: false,
+          error: {
+            code: "screenshot_no_frame",
+            message:
+              "The browser tab has no painted frame. Focus the tab in the app, then try again.",
+            retryable: false,
+          },
+        });
+      } finally {
+        vi.useRealTimers();
+      }
+    });
   });
 
   describe("unsupported command", () => {

packages/desktop/src/features/browser-automation/service.ts

@@ -58,6 +58,9 @@ type FailurePayload = Extract<AutomationCommandPayload, { ok: false }>;
 const defaultSnapshotEngine = new BrowserSnapshotEngine();
 const DEFAULT_WAIT_TIMEOUT_MS = 5_000;
 const WAIT_POLL_INTERVAL_MS = 25;
+const PIXEL_CAPTURE_TIMEOUT_MS = 5_000;
+const SCREENSHOT_NO_FRAME_MESSAGE =
+  "The browser tab has no painted frame. Focus the tab in the app, then try again.";
 const ALLOWED_PAGE_URL_PROTOCOLS = new Set(["http:", "https:"]);
 
 function fail(
@@ -69,6 +72,38 @@ function fail(
   return { requestId, ok: false, error: { code, message, retryable } };
 }
 
+class ScreenshotNoFrameError extends Error {
+  public constructor() {
+    super(SCREENSHOT_NO_FRAME_MESSAGE);
+    this.name = "ScreenshotNoFrameError";
+  }
+}
+
+function isScreenshotNoFrameError(error: unknown): error is ScreenshotNoFrameError {
+  return error instanceof ScreenshotNoFrameError;
+}
+
+function screenshotNoFrameFailure(requestId: string): FailurePayload {
+  return fail(requestId, "screenshot_no_frame", SCREENSHOT_NO_FRAME_MESSAGE);
+}
+
+async function withPixelCaptureTimeout<T>(capture: Promise<T>): Promise<T> {
+  let timeoutId: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<never>((_, reject) => {
+    timeoutId = setTimeout(() => {
+      reject(new ScreenshotNoFrameError());
+    }, PIXEL_CAPTURE_TIMEOUT_MS);
+  });
+
+  try {
+    return await Promise.race([capture, timeout]);
+  } finally {
+    if (timeoutId) {
+      clearTimeout(timeoutId);
+    }
+  }
+}
+
 function tabInfoFromContents(
   browserId: string,
   contents: TabContents,
@@ -1070,10 +1105,20 @@ async function executeScreenshot(
     return target;
   }
   if (target.contents.sendDebugCommand) {
-    const screenshot = (await target.contents.sendDebugCommand("Page.captureScreenshot", {
+    let screenshot: CdpCaptureScreenshotResult;
-      format: "png",
+    try {
-      fromSurface: false,
+      screenshot = (await withPixelCaptureTimeout(
-    })) as CdpCaptureScreenshotResult;
+        target.contents.sendDebugCommand("Page.captureScreenshot", {
+          format: "png",
+          fromSurface: false,
+        }),
+      )) as CdpCaptureScreenshotResult;
+    } catch (error) {
+      if (isScreenshotNoFrameError(error)) {
+        return screenshotNoFrameFailure(requestId);
+      }
+      throw error;
+    }
     if (!screenshot.data) {
       return fail(requestId, "browser_unsupported", "browser_screenshot returned no data");
     }
@@ -1091,7 +1136,15 @@ async function executeScreenshot(
       },
     };
   }
-  const image = await target.contents.capturePage();
+  let image: TabImage;
+  try {
+    image = await withPixelCaptureTimeout(target.contents.capturePage());
+  } catch (error) {
+    if (isScreenshotNoFrameError(error)) {
+      return screenshotNoFrameFailure(requestId);
+    }
+    throw error;
+  }
   const size = image.getSize();
   return {
     requestId,
@@ -1180,11 +1233,21 @@ async function executeFullPageScreenshot(
   const metrics = await getCdpLayoutMetrics(target.contents);
   const width = metrics.contentWidth;
   const height = metrics.contentHeight;
-  const screenshot = (await target.contents.sendDebugCommand("Page.captureScreenshot", {
+  let screenshot: CdpCaptureScreenshotResult;
-    format: "png",
+  try {
-    captureBeyondViewport: true,
+    screenshot = (await withPixelCaptureTimeout(
-    clip: { x: 0, y: 0, width, height, scale: 1 },
+      target.contents.sendDebugCommand("Page.captureScreenshot", {
-  })) as CdpCaptureScreenshotResult;
+        format: "png",
+        captureBeyondViewport: true,
+        clip: { x: 0, y: 0, width, height, scale: 1 },
+      }),
+    )) as CdpCaptureScreenshotResult;
+  } catch (error) {
+    if (isScreenshotNoFrameError(error)) {
+      return screenshotNoFrameFailure(requestId);
+    }
+    throw error;
+  }
   if (!screenshot.data) {
     return fail(requestId, "browser_unsupported", "browser_full_page_screenshot returned no data");
   }

packages/protocol/src/browser-automation/rpc-schemas.test.ts

@@ -374,6 +374,30 @@ describe("browser automation execute RPC schemas", () => {
         height: 50,
       },
     });
+
+    expect(
+      BrowserAutomationExecuteResponseSchema.parse({
+        type: "browser.automation.execute.response",
+        payload: {
+          requestId: "req-shot-no-frame",
+          ok: false,
+          error: {
+            code: "screenshot_no_frame",
+            message:
+              "The browser tab has no painted frame. Focus the tab in the app, then try again.",
+            retryable: false,
+          },
+        },
+      }).payload,
+    ).toEqual({
+      requestId: "req-shot-no-frame",
+      ok: false,
+      error: {
+        code: "screenshot_no_frame",
+        message: "The browser tab has no painted frame. Focus the tab in the app, then try again.",
+        retryable: false,
+      },
+    });
   });
 
   test("parses form control commands and responses", () => {

packages/protocol/src/browser-automation/rpc-schemas.ts

@@ -7,6 +7,7 @@ export const BrowserAutomationErrorCodeSchema = z.enum([
   "browser_tab_not_found",
   "browser_tab_closed",
   "browser_timeout",
+  "screenshot_no_frame",
   "browser_denied",
   "browser_unsupported",
   "browser_stale_ref",

packages/server/src/server/browser-tools/tools.test.ts

@@ -908,4 +908,29 @@ describe("registerBrowserTools", () => {
       retryable: true,
     });
   });
+
+  it("preserves screenshot no-frame broker messages", async () => {
+    const message =
+      "The browser tab has no painted frame. Focus the tab in the app, then try again.";
+    const harness = createHarness({
+      brokerResponse: {
+        requestId: "req-no-frame",
+        ok: false,
+        error: {
+          code: "screenshot_no_frame",
+          message,
+          retryable: false,
+        },
+      },
+    });
+
+    const response = await tool(harness, "browser_screenshot").handler({});
+
+    expect(response.content).toEqual([{ type: "text", text: message }]);
+    expect(response.structuredContent?.error).toEqual({
+      code: "screenshot_no_frame",
+      message,
+      retryable: false,
+    });
+  });
 });

packages/server/src/server/browser-tools/tools.ts

@@ -1116,6 +1116,8 @@ function summarizeBrowserError(
       return "No active browser tab is available. Call browser_new_tab to create one, then omit browserId for the active tab or use the returned browserId.";
     case "browser_timeout":
       return "The browser did not respond before the timeout. Try again or check the desktop app.";
+    case "screenshot_no_frame":
+      return error.message;
     case "browser_unsupported":
       return "This desktop build does not support that browser automation request yet.";
     case "browser_stale_ref":