Use Electron capturePage for browser screenshots

packages/desktop/src/features/browser-automation/ipc.ts

@@ -23,6 +23,7 @@ import {
 const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
 const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
 const observedContentsIds = new Set<number>();
+const backgroundThrottlingByContentsId = new Map<number, boolean>();
 
 interface IpcHandlerRegistry {
   handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void;
@@ -43,7 +44,10 @@ function adaptWebContents(contents: WebContents): TabContents {
     goBack: () => contents.goBack(),
     goForward: () => contents.goForward(),
     reload: () => contents.reload(),
-    capturePage: () => contents.capturePage(),
+    capturePage: (options) => contents.capturePage(undefined, options),
+    invalidate: () => contents.invalidate(),
+    isBackgroundThrottlingAllowed: () => readBackgroundThrottling(contents),
+    setBackgroundThrottling: (allowed) => setBackgroundThrottling(contents, allowed),
     getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
     getCookies: async (url: string) =>
       (await contents.session.cookies.get({ url })).map(normalizeCookie),
@@ -103,6 +107,15 @@ function normalizeCookie(cookie: Electron.Cookie): BrowserAutomationCookieEntry
   };
 }
 
+function readBackgroundThrottling(contents: WebContents): boolean {
+  return backgroundThrottlingByContentsId.get(contents.id) ?? true;
+}
+
+function setBackgroundThrottling(contents: WebContents, allowed: boolean): void {
+  backgroundThrottlingByContentsId.set(contents.id, allowed);
+  contents.setBackgroundThrottling(allowed);
+}
+
 function observeConsoleMessages(contents: WebContents): void {
   if (observedContentsIds.has(contents.id)) {
     return;
@@ -117,6 +130,7 @@ function observeConsoleMessages(contents: WebContents): void {
   contents.once("destroyed", () => {
     observedContentsIds.delete(contents.id);
     consoleMessagesByContentsId.delete(contents.id);
+    backgroundThrottlingByContentsId.delete(contents.id);
   });
 }
 

packages/desktop/src/features/browser-automation/service.test.ts

@@ -22,6 +22,9 @@ function fakeTab(overrides: Partial<TabContents> & { id: number }): TabContents
       toPNG: () => new Uint8Array([137, 80, 78, 71]),
       getSize: () => ({ width: 10, height: 5 }),
     }),
+    invalidate: () => {},
+    isBackgroundThrottlingAllowed: () => true,
+    setBackgroundThrottling: () => {},
     ...overrides,
   };
 }
@@ -1932,20 +1935,27 @@ describe("executeAutomationCommand", () => {
   });
 
   describe("screenshot", () => {
-    it("captures a viewport screenshot through CDP when available", async () => {
+    it("captures a viewport screenshot through Electron capturePage when CDP is available", async () => {
-      let captureParams: Record<string, unknown> | undefined;
+      const actions: string[] = [];
+      let captureOptions: unknown;
       const tab = fakeTab({
         id: 13,
-        executeJavaScript: async () => ({ width: 321, height: 123 }),
+        sendDebugCommand: async (command) => {
-        sendDebugCommand: async (command, params) => {
-          if (command === "Page.captureScreenshot") {
-            captureParams = params;
-            return { data: "iVBORw0KGgo=" };
-          }
           throw new Error(`Unexpected CDP command ${command}`);
         },
-        capturePage: async () => {
+        capturePage: async (options) => {
-          throw new Error("capturePage should not be used when CDP is available");
+          actions.push("capture");
+          captureOptions = options;
+          return {
+            toPNG: () => new Uint8Array([137, 80, 78, 71, 1, 2, 3]),
+            getSize: () => ({ width: 640, height: 480 }),
+          };
+        },
+        invalidate: () => {
+          actions.push("invalidate");
+        },
+        setBackgroundThrottling: (allowed) => {
+          actions.push(`background:${allowed}`);
         },
       });
       const registry = createRegistry({
@@ -1958,7 +1968,7 @@ describe("executeAutomationCommand", () => {
       const result = await executeAutomationCommand(
         {
           type: "browser.automation.execute.request",
-          requestId: "r-screenshot-cdp",
+          requestId: "r-screenshot-capture-page",
           workspaceId: "workspace-a",
           command: { command: "screenshot", args: { workspaceId: "workspace-a" } },
         },
@@ -1966,33 +1976,42 @@ describe("executeAutomationCommand", () => {
       );
 
       expect(result).toEqual({
-        requestId: "r-screenshot-cdp",
+        requestId: "r-screenshot-capture-page",
         ok: true,
         result: {
           command: "screenshot",
           browserId: "a",
           mimeType: "image/png",
-          dataBase64: "iVBORw0KGgo=",
+          dataBase64: "iVBORwECAw==",
-          width: 321,
+          width: 640,
-          height: 123,
+          height: 480,
         },
       });
-      expect(captureParams).toEqual({ format: "png", fromSurface: false });
+      expect(captureOptions).toEqual({ stayHidden: false });
+      expect(actions).toEqual(["background:false", "invalidate", "capture", "background:true"]);
     });
 
-    it("returns screenshot_no_frame when CDP viewport capture never paints", async () => {
+    it("returns screenshot_no_frame when viewport capture never paints", async () => {
       vi.useFakeTimers();
       try {
+        const actions: string[] = [];
+        let backgroundThrottlingAllowed = false;
         const tab = fakeTab({
           id: 13,
           sendDebugCommand: async (command) => {
-            if (command === "Page.captureScreenshot") {
-              return new Promise<never>(() => {});
-            }
             throw new Error(`Unexpected CDP command ${command}`);
           },
           capturePage: async () => {
-            throw new Error("capturePage should not be used when CDP is available");
+            actions.push("capture");
+            return new Promise<never>(() => {});
+          },
+          invalidate: () => {
+            actions.push("invalidate");
+          },
+          isBackgroundThrottlingAllowed: () => backgroundThrottlingAllowed,
+          setBackgroundThrottling: (allowed) => {
+            backgroundThrottlingAllowed = allowed;
+            actions.push(`background:${allowed}`);
           },
         });
         const registry = createRegistry({
@@ -2006,7 +2025,7 @@ describe("executeAutomationCommand", () => {
         const resultPromise = executeAutomationCommand(
           {
             type: "browser.automation.execute.request",
-            requestId: "r-screenshot-cdp-no-frame",
+            requestId: "r-screenshot-no-frame",
             workspaceId: "workspace-a",
             command: { command: "screenshot", args: { workspaceId: "workspace-a" } },
           },
@@ -2016,7 +2035,7 @@ describe("executeAutomationCommand", () => {
         await vi.advanceTimersByTimeAsync(5_000);
 
         await expect(resultPromise).resolves.toEqual({
-          requestId: "r-screenshot-cdp-no-frame",
+          requestId: "r-screenshot-no-frame",
           ok: false,
           error: {
             code: "screenshot_no_frame",
@@ -2025,6 +2044,8 @@ describe("executeAutomationCommand", () => {
             retryable: false,
           },
         });
+        expect(backgroundThrottlingAllowed).toBe(false);
+        expect(actions).toEqual(["background:false", "invalidate", "capture", "background:false"]);
       } finally {
         vi.useRealTimers();
       }

packages/desktop/src/features/browser-automation/service.ts

@@ -25,7 +25,10 @@ export interface TabContents {
   goBack(): void;
   goForward(): void;
   reload(): void;
-  capturePage(): Promise<TabImage>;
+  capturePage(options?: TabCapturePageOptions): Promise<TabImage>;
+  invalidate(): void;
+  isBackgroundThrottlingAllowed(): boolean;
+  setBackgroundThrottling(allowed: boolean): void;
   getConsoleMessages?(): BrowserAutomationConsoleLogEntry[];
   getCookies?(url: string): Promise<BrowserAutomationCookieEntry[]>;
   sendDebugCommand?(command: string, params?: Record<string, unknown>): Promise<unknown>;
@@ -42,6 +45,10 @@ export interface TabImage {
   getSize(): { width: number; height: number };
 }
 
+export interface TabCapturePageOptions {
+  stayHidden?: boolean;
+}
+
 export interface BrowserRegistry {
   listRegisteredBrowserIds(): string[];
   listRegisteredBrowserIdsForWorkspace(workspaceId: string): string[];
@@ -104,6 +111,17 @@ async function withPixelCaptureTimeout<T>(capture: Promise<T>): Promise<T> {
   }
 }
 
+async function capturePaintedViewport(contents: TabContents): Promise<TabImage> {
+  const previousBackgroundThrottling = contents.isBackgroundThrottlingAllowed();
+  contents.setBackgroundThrottling(false);
+  try {
+    contents.invalidate();
+    return await withPixelCaptureTimeout(contents.capturePage({ stayHidden: false }));
+  } finally {
+    contents.setBackgroundThrottling(previousBackgroundThrottling);
+  }
+}
+
 function tabInfoFromContents(
   browserId: string,
   contents: TabContents,
@@ -1104,41 +1122,9 @@ async function executeScreenshot(
   if ("ok" in target) {
     return target;
   }
-  if (target.contents.sendDebugCommand) {
-    let screenshot: CdpCaptureScreenshotResult;
-    try {
-      screenshot = (await withPixelCaptureTimeout(
-        target.contents.sendDebugCommand("Page.captureScreenshot", {
-          format: "png",
-          fromSurface: false,
-        }),
-      )) as CdpCaptureScreenshotResult;
-    } catch (error) {
-      if (isScreenshotNoFrameError(error)) {
-        return screenshotNoFrameFailure(requestId);
-      }
-      throw error;
-    }
-    if (!screenshot.data) {
-      return fail(requestId, "browser_unsupported", "browser_screenshot returned no data");
-    }
-    const viewport = await getViewportSize(target.contents);
-    return {
-      requestId,
-      ok: true,
-      result: {
-        command: "screenshot",
-        browserId: target.browserId,
-        mimeType: "image/png",
-        dataBase64: screenshot.data,
-        width: viewport.width,
-        height: viewport.height,
-      },
-    };
-  }
   let image: TabImage;
   try {
-    image = await withPixelCaptureTimeout(target.contents.capturePage());
+    image = await capturePaintedViewport(target.contents);
   } catch (error) {
     if (isScreenshotNoFrameError(error)) {
       return screenshotNoFrameFailure(requestId);
@@ -1183,20 +1169,6 @@ interface CdpCaptureScreenshotResult {
   data?: string;
 }
 
-async function getViewportSize(contents: TabContents): Promise<{ width: number; height: number }> {
-  const result = await contents.executeJavaScript(
-    "({ width: Math.round(window.innerWidth), height: Math.round(window.innerHeight) })",
-  );
-  if (!result || typeof result !== "object") {
-    return { width: 0, height: 0 };
-  }
-  const record = result as { width?: unknown; height?: unknown };
-  return {
-    width: typeof record.width === "number" && Number.isFinite(record.width) ? record.width : 0,
-    height: typeof record.height === "number" && Number.isFinite(record.height) ? record.height : 0,
-  };
-}
-
 async function getCdpLayoutMetrics(contents: TabContents): Promise<{
   viewportWidth: number;
   viewportHeight: number;