From 6fea8305a7e834380c2310dcdc01bd190d08e00b Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Thu, 2 Jul 2026 18:53:57 +0200 Subject: [PATCH] Use Electron capturePage for browser screenshots --- .../src/features/browser-automation/ipc.ts | 16 ++++- .../browser-automation/service.test.ts | 67 +++++++++++------- .../features/browser-automation/service.ts | 68 ++++++------------- 3 files changed, 79 insertions(+), 72 deletions(-) diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts index 92c716d46..54295f96e 100644 --- a/packages/desktop/src/features/browser-automation/ipc.ts +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -23,6 +23,7 @@ import { const MAX_CONSOLE_MESSAGES_PER_TAB = 200; const consoleMessagesByContentsId = new Map(); const observedContentsIds = new Set(); +const backgroundThrottlingByContentsId = new Map(); interface IpcHandlerRegistry { handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void; @@ -43,7 +44,10 @@ function adaptWebContents(contents: WebContents): TabContents { goBack: () => contents.goBack(), goForward: () => contents.goForward(), reload: () => contents.reload(), - capturePage: () => contents.capturePage(), + capturePage: (options) => contents.capturePage(undefined, options), + invalidate: () => contents.invalidate(), + isBackgroundThrottlingAllowed: () => readBackgroundThrottling(contents), + setBackgroundThrottling: (allowed) => setBackgroundThrottling(contents, allowed), getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [], getCookies: async (url: string) => (await contents.session.cookies.get({ url })).map(normalizeCookie), @@ -103,6 +107,15 @@ function normalizeCookie(cookie: Electron.Cookie): BrowserAutomationCookieEntry }; } +function readBackgroundThrottling(contents: WebContents): boolean { + return backgroundThrottlingByContentsId.get(contents.id) ?? true; +} + +function setBackgroundThrottling(contents: WebContents, allowed: boolean): void { + backgroundThrottlingByContentsId.set(contents.id, allowed); + contents.setBackgroundThrottling(allowed); +} + function observeConsoleMessages(contents: WebContents): void { if (observedContentsIds.has(contents.id)) { return; @@ -117,6 +130,7 @@ function observeConsoleMessages(contents: WebContents): void { contents.once("destroyed", () => { observedContentsIds.delete(contents.id); consoleMessagesByContentsId.delete(contents.id); + backgroundThrottlingByContentsId.delete(contents.id); }); } diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index f35ab5fd7..c01a50c17 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -22,6 +22,9 @@ function fakeTab(overrides: Partial & { id: number }): TabContents toPNG: () => new Uint8Array([137, 80, 78, 71]), getSize: () => ({ width: 10, height: 5 }), }), + invalidate: () => {}, + isBackgroundThrottlingAllowed: () => true, + setBackgroundThrottling: () => {}, ...overrides, }; } @@ -1932,20 +1935,27 @@ describe("executeAutomationCommand", () => { }); describe("screenshot", () => { - it("captures a viewport screenshot through CDP when available", async () => { - let captureParams: Record | undefined; + it("captures a viewport screenshot through Electron capturePage when CDP is available", async () => { + const actions: string[] = []; + let captureOptions: unknown; const tab = fakeTab({ id: 13, - executeJavaScript: async () => ({ width: 321, height: 123 }), - sendDebugCommand: async (command, params) => { - if (command === "Page.captureScreenshot") { - captureParams = params; - return { data: "iVBORw0KGgo=" }; - } + sendDebugCommand: async (command) => { throw new Error(`Unexpected CDP command ${command}`); }, - capturePage: async () => { - throw new Error("capturePage should not be used when CDP is available"); + capturePage: async (options) => { + actions.push("capture"); + captureOptions = options; + return { + toPNG: () => new Uint8Array([137, 80, 78, 71, 1, 2, 3]), + getSize: () => ({ width: 640, height: 480 }), + }; + }, + invalidate: () => { + actions.push("invalidate"); + }, + setBackgroundThrottling: (allowed) => { + actions.push(`background:${allowed}`); }, }); const registry = createRegistry({ @@ -1958,7 +1968,7 @@ describe("executeAutomationCommand", () => { const result = await executeAutomationCommand( { type: "browser.automation.execute.request", - requestId: "r-screenshot-cdp", + requestId: "r-screenshot-capture-page", workspaceId: "workspace-a", command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, }, @@ -1966,33 +1976,42 @@ describe("executeAutomationCommand", () => { ); expect(result).toEqual({ - requestId: "r-screenshot-cdp", + requestId: "r-screenshot-capture-page", ok: true, result: { command: "screenshot", browserId: "a", mimeType: "image/png", - dataBase64: "iVBORw0KGgo=", - width: 321, - height: 123, + dataBase64: "iVBORwECAw==", + width: 640, + height: 480, }, }); - expect(captureParams).toEqual({ format: "png", fromSurface: false }); + expect(captureOptions).toEqual({ stayHidden: false }); + expect(actions).toEqual(["background:false", "invalidate", "capture", "background:true"]); }); - it("returns screenshot_no_frame when CDP viewport capture never paints", async () => { + it("returns screenshot_no_frame when viewport capture never paints", async () => { vi.useFakeTimers(); try { + const actions: string[] = []; + let backgroundThrottlingAllowed = false; const tab = fakeTab({ id: 13, sendDebugCommand: async (command) => { - if (command === "Page.captureScreenshot") { - return new Promise(() => {}); - } throw new Error(`Unexpected CDP command ${command}`); }, capturePage: async () => { - throw new Error("capturePage should not be used when CDP is available"); + actions.push("capture"); + return new Promise(() => {}); + }, + invalidate: () => { + actions.push("invalidate"); + }, + isBackgroundThrottlingAllowed: () => backgroundThrottlingAllowed, + setBackgroundThrottling: (allowed) => { + backgroundThrottlingAllowed = allowed; + actions.push(`background:${allowed}`); }, }); const registry = createRegistry({ @@ -2006,7 +2025,7 @@ describe("executeAutomationCommand", () => { const resultPromise = executeAutomationCommand( { type: "browser.automation.execute.request", - requestId: "r-screenshot-cdp-no-frame", + requestId: "r-screenshot-no-frame", workspaceId: "workspace-a", command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, }, @@ -2016,7 +2035,7 @@ describe("executeAutomationCommand", () => { await vi.advanceTimersByTimeAsync(5_000); await expect(resultPromise).resolves.toEqual({ - requestId: "r-screenshot-cdp-no-frame", + requestId: "r-screenshot-no-frame", ok: false, error: { code: "screenshot_no_frame", @@ -2025,6 +2044,8 @@ describe("executeAutomationCommand", () => { retryable: false, }, }); + expect(backgroundThrottlingAllowed).toBe(false); + expect(actions).toEqual(["background:false", "invalidate", "capture", "background:false"]); } finally { vi.useRealTimers(); } diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index 780a4b864..b7138251f 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -25,7 +25,10 @@ export interface TabContents { goBack(): void; goForward(): void; reload(): void; - capturePage(): Promise; + capturePage(options?: TabCapturePageOptions): Promise; + invalidate(): void; + isBackgroundThrottlingAllowed(): boolean; + setBackgroundThrottling(allowed: boolean): void; getConsoleMessages?(): BrowserAutomationConsoleLogEntry[]; getCookies?(url: string): Promise; sendDebugCommand?(command: string, params?: Record): Promise; @@ -42,6 +45,10 @@ export interface TabImage { getSize(): { width: number; height: number }; } +export interface TabCapturePageOptions { + stayHidden?: boolean; +} + export interface BrowserRegistry { listRegisteredBrowserIds(): string[]; listRegisteredBrowserIdsForWorkspace(workspaceId: string): string[]; @@ -104,6 +111,17 @@ async function withPixelCaptureTimeout(capture: Promise): Promise { } } +async function capturePaintedViewport(contents: TabContents): Promise { + const previousBackgroundThrottling = contents.isBackgroundThrottlingAllowed(); + contents.setBackgroundThrottling(false); + try { + contents.invalidate(); + return await withPixelCaptureTimeout(contents.capturePage({ stayHidden: false })); + } finally { + contents.setBackgroundThrottling(previousBackgroundThrottling); + } +} + function tabInfoFromContents( browserId: string, contents: TabContents, @@ -1104,41 +1122,9 @@ async function executeScreenshot( if ("ok" in target) { return target; } - if (target.contents.sendDebugCommand) { - let screenshot: CdpCaptureScreenshotResult; - try { - screenshot = (await withPixelCaptureTimeout( - target.contents.sendDebugCommand("Page.captureScreenshot", { - format: "png", - fromSurface: false, - }), - )) as CdpCaptureScreenshotResult; - } catch (error) { - if (isScreenshotNoFrameError(error)) { - return screenshotNoFrameFailure(requestId); - } - throw error; - } - if (!screenshot.data) { - return fail(requestId, "browser_unsupported", "browser_screenshot returned no data"); - } - const viewport = await getViewportSize(target.contents); - return { - requestId, - ok: true, - result: { - command: "screenshot", - browserId: target.browserId, - mimeType: "image/png", - dataBase64: screenshot.data, - width: viewport.width, - height: viewport.height, - }, - }; - } let image: TabImage; try { - image = await withPixelCaptureTimeout(target.contents.capturePage()); + image = await capturePaintedViewport(target.contents); } catch (error) { if (isScreenshotNoFrameError(error)) { return screenshotNoFrameFailure(requestId); @@ -1183,20 +1169,6 @@ interface CdpCaptureScreenshotResult { data?: string; } -async function getViewportSize(contents: TabContents): Promise<{ width: number; height: number }> { - const result = await contents.executeJavaScript( - "({ width: Math.round(window.innerWidth), height: Math.round(window.innerHeight) })", - ); - if (!result || typeof result !== "object") { - return { width: 0, height: 0 }; - } - const record = result as { width?: unknown; height?: unknown }; - return { - width: typeof record.width === "number" && Number.isFinite(record.width) ? record.width : 0, - height: typeof record.height === "number" && Number.isFinite(record.height) ? record.height : 0, - }; -} - async function getCdpLayoutMetrics(contents: TabContents): Promise<{ viewportWidth: number; viewportHeight: number;