diff --git a/docs/browser-capture-harness.md b/docs/browser-capture-harness.md index 9c8009dfb..c598f909f 100644 --- a/docs/browser-capture-harness.md +++ b/docs/browser-capture-harness.md @@ -8,8 +8,8 @@ It validates the compositor behavior that unit tests cannot see: - the resident webview guest is sized to 1280x800 logical pixels; - multiple resident webviews are parked as an overlapping stack, and the capture target is raised above its sibling webviews before capture; -- a newly attached resident webview can be captured immediately after load, without - an extra settle delay; +- a newly attached resident webview whose first useful frame is delayed can be captured + by holding prep active and retrying until the frame appears; - the app-style prep sequence moves the host to a paintable 1x1 clipped state, waits two animation frames and a layout read, then both viewport `capturePage` and full-page CDP screenshots return real pixels; - restore returns the host to offscreen parking after every capture. @@ -27,10 +27,10 @@ packages/desktop/capture-harness/out/ A passing run prints `PASS` lines for both guest sizes, the expected parked-capture failure, the legacy stacked-below-the-clip failure for the second webview, five viewport -prep captures and five full-page prep captures for each of the first two webviews, a -fresh-immediate viewport and full-page capture for a newly attached webview, and final -completion. The PNG sizes may be device-pixel scaled; on a Retina display the 1280x800 -logical viewport is usually saved as 2560x1600. +prep captures and five full-page prep captures for each of the first two webviews, +fresh-delayed-first-frame viewport and full-page captures for a newly attached webview, +and final completion. The PNG sizes may be device-pixel scaled; on a Retina display the +1280x800 logical viewport is usually saved as 2560x1600. ## Mechanism diff --git a/packages/desktop/capture-harness/delayed-bright.html b/packages/desktop/capture-harness/delayed-bright.html new file mode 100644 index 000000000..67c5c7fae --- /dev/null +++ b/packages/desktop/capture-harness/delayed-bright.html @@ -0,0 +1,93 @@ + + + + + Paseo Delayed Capture Harness Target + + + + +
+
PASEO CAPTURE HARNESS
+
DELAYED FIRST PAINT TARGET
+
+
FULL PAGE MARKER AT Y=1370
+
+ + diff --git a/packages/desktop/capture-harness/main.js b/packages/desktop/capture-harness/main.js index 8523bab22..115265165 100644 --- a/packages/desktop/capture-harness/main.js +++ b/packages/desktop/capture-harness/main.js @@ -9,6 +9,7 @@ const VIEWPORT_WIDTH = 1280; const VIEWPORT_HEIGHT = 800; const FULL_PAGE_HEIGHT = 1600; const CAPTURE_TIMEOUT_MS = 5000; +const CAPTURE_RETRY_INTERVAL_MS = 200; const REPEAT_COUNT = 5; function fileUrl(filePath) { @@ -143,6 +144,10 @@ async function saveImage(image, outputPath) { await fsp.writeFile(outputPath, image.toPNG()); } +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + async function waitForGuestLoad(contents, input = {}) { await new Promise((resolve) => { if (!contents.isLoading()) { @@ -240,6 +245,7 @@ async function captureWithPrep({ guestMetrics, repeatTotal = REPEAT_COUNT, label = "prep", + retryUntilPass = false, }) { const preparation = await renderer( win, @@ -250,26 +256,43 @@ async function captureWithPrep({ `${mode}-webview-${targetIndex + 1}-${label}-${repeatIndex}.png`, ); try { - const image = - mode === "viewport" ? await capturePageSequence(contents) : await captureFullPage(contents); - await saveImage(image, outputPath); const expected = mode === "viewport" ? { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 } : { width: VIEWPORT_WIDTH, height: FULL_PAGE_HEIGHT, minBrightRatio: 0.55 }; - const analysis = analyzeImage(image, expected, guestMetrics); - const size = `${analysis.width}x${analysis.height}`; - const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`; - const bright = analysis.brightRatio.toFixed(4); - if (!analysis.pass) { - fail( - `${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`, - ); + const deadline = Date.now() + CAPTURE_TIMEOUT_MS; + let attempt = 0; + let lastFailure = "capture did not run"; + while (Date.now() < deadline) { + attempt += 1; + try { + const image = + mode === "viewport" + ? await capturePageSequence(contents) + : await captureFullPage(contents); + const analysis = analyzeImage(image, expected, guestMetrics); + const size = `${analysis.width}x${analysis.height}`; + const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`; + const bright = analysis.brightRatio.toFixed(4); + if (analysis.pass) { + await saveImage(image, outputPath); + pass( + `${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} attempts=${attempt} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`, + ); + return analysis; + } + lastFailure = `size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform}`; + } catch (error) { + lastFailure = error instanceof Error ? error.message : String(error); + } + if (!retryUntilPass) { + break; + } + await delay(Math.min(CAPTURE_RETRY_INTERVAL_MS, Math.max(0, deadline - Date.now()))); } - pass( - `${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`, + fail( + `${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} failed attempts=${attempt} last=${lastFailure} file=${outputPath}`, ); - return analysis; } finally { const restoredState = await renderer( win, @@ -320,14 +343,13 @@ async function expectLegacySecondWebviewFailure({ win, contents, mode, guestMetr } } -async function captureFreshWebviewImmediately({ win, waitForNextAttachedGuest }) { +async function captureFreshDelayedWebview({ win, waitForNextAttachedGuest, mode }) { const freshGuestPromise = waitForNextAttachedGuest(); const targetIndex = await renderer( win, - `window.captureHarness.addWebview(${JSON.stringify(fileUrl(path.join(ROOT, "bright.html")))})`, + `window.captureHarness.addWebview(${JSON.stringify(fileUrl(path.join(ROOT, "delayed-bright.html")))})`, ); const guest = await withTimeout(freshGuestPromise, "fresh did-attach-webview"); - await waitForGuestLoad(guest, { settleMs: 0 }); const guestMetrics = await readGuestMetrics(guest); if (guestMetrics.innerWidth !== VIEWPORT_WIDTH || guestMetrics.innerHeight !== VIEWPORT_HEIGHT) { fail( @@ -340,22 +362,13 @@ async function captureFreshWebviewImmediately({ win, waitForNextAttachedGuest }) await captureWithPrep({ win, contents: guest, - mode: "viewport", + mode, repeatIndex: 1, targetIndex, guestMetrics, repeatTotal: 1, - label: "fresh-immediate", - }); - await captureWithPrep({ - win, - contents: guest, - mode: "full-page", - repeatIndex: 1, - targetIndex, - guestMetrics, - repeatTotal: 1, - label: "fresh-immediate", + label: "fresh-delayed-first-frame", + retryUntilPass: true, }); } @@ -458,7 +471,9 @@ async function main() { await renderer(win, "window.captureHarness.restoreParking()"); - await captureFreshWebviewImmediately({ win, waitForNextAttachedGuest }); + await captureFreshDelayedWebview({ win, waitForNextAttachedGuest, mode: "viewport" }); + await renderer(win, "window.captureHarness.restoreParking()"); + await captureFreshDelayedWebview({ win, waitForNextAttachedGuest, mode: "full-page" }); await renderer(win, "window.captureHarness.restoreParking()"); const results = []; diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index 8006ddd44..28ef2c598 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -54,11 +54,13 @@ class FakeTab implements TabContents { public captureNeverPaints = false; public captureThrows = false; public captureErrorMessage = "capture failed"; + public viewportCaptureFailuresBeforeSuccess = 0; public deferCaptures = false; public prepareNeverAcks = false; public prepareErrorMessage: string | null = null; public fullPageScreenshotThrows = false; public fullPageScreenshotErrorMessage = "UnknownVizError"; + public fullPageCaptureFailuresBeforeSuccess = 0; public layoutMetrics = { cssLayoutViewport: { clientWidth: 390, clientHeight: 844 }, cssContentSize: { width: 390, height: 1200 }, @@ -133,6 +135,10 @@ class FakeTab implements TabContents { this.capturedViewports.push(options ?? {}); this.actions.push("capture"); this.resolveCaptureStartWaiters(); + if (this.viewportCaptureFailuresBeforeSuccess > 0) { + this.viewportCaptureFailuresBeforeSuccess -= 1; + throw new Error(this.captureErrorMessage); + } if (this.captureThrows) { throw new Error(this.captureErrorMessage); } @@ -191,6 +197,10 @@ class FakeTab implements TabContents { return this.layoutMetrics; } if (command === "Page.captureScreenshot") { + if (this.fullPageCaptureFailuresBeforeSuccess > 0) { + this.fullPageCaptureFailuresBeforeSuccess -= 1; + throw new Error(this.fullPageScreenshotErrorMessage); + } if (this.fullPageScreenshotThrows) { throw new Error(this.fullPageScreenshotErrorMessage); } @@ -990,34 +1000,82 @@ describe("executeAutomationCommand", () => { ]); }); - test("screenshot returns no-frame and restores capture preparation when viewport capture throws UnknownVizError", async () => { - const browser = new BrowserAutomationHarness(); - browser.tab.captureThrows = true; - browser.tab.captureErrorMessage = "UnknownVizError"; + test("screenshot retries UnknownVizError until the first viewport frame appears", async () => { + vi.useFakeTimers(); + try { + const browser = new BrowserAutomationHarness(); + browser.tab.captureErrorMessage = "UnknownVizError"; + browser.tab.viewportCaptureFailuresBeforeSuccess = 2; - await expect( - browser.execute({ + const resultPromise = browser.execute({ command: "screenshot", args: { browserId: BROWSER_A }, - }), - ).resolves.toEqual({ - requestId: "req-screenshot", - ok: false, - error: { - code: "screenshot_no_frame", - message: "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, - }, - }); + }); + await vi.advanceTimersByTimeAsync(400); - expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", - "invalidate", - "capture", - "restore:capture-1", - "background:true", - ]); + await expect(resultPromise).resolves.toEqual({ + requestId: "req-screenshot", + ok: true, + result: { + command: "screenshot", + browserId: BROWSER_A, + mimeType: "image/png", + dataBase64: "iVBORwECAw==", + width: 640, + height: 480, + }, + }); + expect(browser.tab.capturedViewports).toHaveLength(3); + expect(browser.tab.actions).toEqual([ + "prepare", + "background:false", + "invalidate", + "capture", + "invalidate", + "capture", + "invalidate", + "capture", + "restore:capture-1", + "background:true", + ]); + } finally { + vi.useRealTimers(); + } + }); + + test("screenshot returns no-frame after UnknownVizError spends the retry budget", async () => { + vi.useFakeTimers(); + try { + const browser = new BrowserAutomationHarness(); + browser.tab.captureThrows = true; + browser.tab.captureErrorMessage = "UnknownVizError"; + + const resultPromise = browser.execute({ + command: "screenshot", + args: { browserId: BROWSER_A }, + }); + await vi.advanceTimersByTimeAsync(5_000); + + await expect(resultPromise).resolves.toEqual({ + requestId: "req-screenshot", + ok: false, + error: { + code: "screenshot_no_frame", + message: + "The browser tab has no painted frame. Focus the tab in the app, then try again.", + retryable: false, + }, + }); + expect(browser.tab.capturedViewports.length).toBeGreaterThan(1); + expect(browser.tab.actions.filter((action) => action === "invalidate")).toHaveLength( + browser.tab.capturedViewports.length, + ); + expect(browser.tab.restoredPixelCaptureTokens).toEqual(["capture-1"]); + expect(browser.tab.actions.at(-2)).toBe("restore:capture-1"); + expect(browser.tab.actions.at(-1)).toBe("background:true"); + } finally { + vi.useRealTimers(); + } }); test("screenshot returns no-frame when capture preparation does not ack", async () => { @@ -1256,33 +1314,45 @@ describe("executeAutomationCommand", () => { ]); }); - test("screenshot with fullPage returns no-frame when CDP capture throws", async () => { - const browser = new BrowserAutomationHarness(); - browser.tab.fullPageScreenshotThrows = true; + test("screenshot with fullPage retries UnknownVizError until the first CDP frame appears", async () => { + vi.useFakeTimers(); + try { + const browser = new BrowserAutomationHarness(); + browser.tab.fullPageCaptureFailuresBeforeSuccess = 1; - const result = await browser.execute({ - command: "screenshot", - args: { browserId: BROWSER_A, fullPage: true }, - }); + const resultPromise = browser.execute({ + command: "screenshot", + args: { browserId: BROWSER_A, fullPage: true }, + }); + await vi.advanceTimersByTimeAsync(200); - expect(result).toEqual({ - requestId: "req-screenshot", - ok: false, - error: { - code: "screenshot_no_frame", - message: "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, - }, - }); - expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", - "invalidate", - "debug:Page.getLayoutMetrics", - "debug:Page.captureScreenshot", - "restore:capture-1", - "background:true", - ]); + await expect(resultPromise).resolves.toEqual({ + requestId: "req-screenshot", + ok: true, + result: { + command: "screenshot", + browserId: BROWSER_A, + mimeType: "image/png", + dataBase64: "fullPagePng", + width: 390, + height: 1200, + }, + }); + expect(browser.tab.actions).toEqual([ + "prepare", + "background:false", + "invalidate", + "debug:Page.getLayoutMetrics", + "debug:Page.captureScreenshot", + "invalidate", + "debug:Page.getLayoutMetrics", + "debug:Page.captureScreenshot", + "restore:capture-1", + "background:true", + ]); + } finally { + vi.useRealTimers(); + } }); test("screenshot with fullPage restores capture preparation when CDP capture fails with an ordinary error", async () => { diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index 06f3fb4f3..3bf4b7838 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -61,6 +61,7 @@ const defaultSnapshotEngine = new BrowserSnapshotEngine(); const DEFAULT_WAIT_TIMEOUT_MS = 5_000; const WAIT_POLL_INTERVAL_MS = 25; const PIXEL_CAPTURE_TIMEOUT_MS = 5_000; +const PIXEL_CAPTURE_RETRY_INTERVAL_MS = 200; const SCREENSHOT_NO_FRAME_MESSAGE = "The browser tab has no painted frame. Focus the tab in the app, then try again."; const SCREENSHOT_PREP_UNAVAILABLE_PREFIX = "Browser screenshot prep_unavailable:"; @@ -94,12 +95,18 @@ function screenshotNoFrameFailure( return fail(requestId, "screenshot_no_frame", error.message); } -async function withPixelCaptureTimeout(capture: Promise): Promise { +async function withPixelCaptureTimeout( + capture: Promise, + timeoutMs = PIXEL_CAPTURE_TIMEOUT_MS, +): Promise { + if (timeoutMs <= 0) { + throw new ScreenshotNoFrameError(); + } let timeoutId: ReturnType | undefined; const timeout = new Promise((_, reject) => { timeoutId = setTimeout(() => { reject(new ScreenshotNoFrameError()); - }, PIXEL_CAPTURE_TIMEOUT_MS); + }, timeoutMs); }); try { @@ -150,18 +157,26 @@ async function restorePixelCapture( } } -async function capturePreparedPixelFrame(capture: () => Promise): Promise { - try { - return await withPixelCaptureTimeout(capture()); - } catch (error) { - if (isScreenshotNoFrameError(error)) { - throw error; +async function capturePreparedPixelFrame( + contents: TabContents, + capture: () => Promise, +): Promise { + const deadline = Date.now() + PIXEL_CAPTURE_TIMEOUT_MS; + while (Date.now() < deadline) { + try { + contents.invalidate(); + return await withPixelCaptureTimeout(capture(), deadline - Date.now()); + } catch (error) { + if (isScreenshotNoFrameError(error)) { + throw error; + } + if (!isKnownNoFrameCaptureError(error)) { + throw error; + } + await delay(Math.min(PIXEL_CAPTURE_RETRY_INTERVAL_MS, Math.max(0, deadline - Date.now()))); } - if (isKnownNoFrameCaptureError(error)) { - throw new ScreenshotNoFrameError(); - } - throw error; } + throw new ScreenshotNoFrameError(); } function screenshotPreparationError(error: unknown): ScreenshotNoFrameError { @@ -206,8 +221,7 @@ async function runPreparedPixelCapture( // paintable before capture; see docs/browser-capture-harness.md. preparation = await prepareForPixelCapture(contents); contents.setBackgroundThrottling(false); - contents.invalidate(); - return await capturePreparedPixelFrame(capture); + return await capturePreparedPixelFrame(contents, capture); } finally { try { if (preparation) {