Retry browser screenshots until first frame

docs/browser-capture-harness.md

@@ -8,8 +8,8 @@ It validates the compositor behavior that unit tests cannot see:
 - the resident webview guest is sized to 1280x800 logical pixels;
 - multiple resident webviews are parked as an overlapping stack, and the capture
   target is raised above its sibling webviews before capture;
-- a newly attached resident webview can be captured immediately after load, without
+- a newly attached resident webview whose first useful frame is delayed can be captured
-  an extra settle delay;
+  by holding prep active and retrying until the frame appears;
 - the app-style prep sequence moves the host to a paintable 1x1 clipped state, waits two animation frames and a layout read, then both viewport `capturePage` and full-page CDP screenshots return real pixels;
 - restore returns the host to offscreen parking after every capture.
 
@@ -27,10 +27,10 @@ packages/desktop/capture-harness/out/
 
 A passing run prints `PASS` lines for both guest sizes, the expected parked-capture
 failure, the legacy stacked-below-the-clip failure for the second webview, five viewport
-prep captures and five full-page prep captures for each of the first two webviews, a
+prep captures and five full-page prep captures for each of the first two webviews,
-fresh-immediate viewport and full-page capture for a newly attached webview, and final
+fresh-delayed-first-frame viewport and full-page captures for a newly attached webview,
-completion. The PNG sizes may be device-pixel scaled; on a Retina display the 1280x800
+and final completion. The PNG sizes may be device-pixel scaled; on a Retina display the
-logical viewport is usually saved as 2560x1600.
+1280x800 logical viewport is usually saved as 2560x1600.
 
 ## Mechanism
 

packages/desktop/capture-harness/delayed-bright.html

@@ -0,0 +1,93 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Paseo Delayed Capture Harness Target</title>
+    <style>
+      html,
+      body {
+        width: 1280px;
+        min-height: 1600px;
+        margin: 0;
+        background: #202020;
+        overflow: auto;
+        font-family: Arial, Helvetica, sans-serif;
+      }
+
+      .content {
+        opacity: 0;
+      }
+
+      body.ready {
+        background: #ff00ff;
+      }
+
+      body.ready .content {
+        opacity: 1;
+      }
+
+      .label {
+        position: absolute;
+        left: 48px;
+        top: 48px;
+        color: #ffffff;
+        font-size: 72px;
+        font-weight: 800;
+        line-height: 1;
+        text-shadow:
+          5px 5px 0 #000000,
+          -2px -2px 0 #000000;
+      }
+
+      .sub {
+        position: absolute;
+        left: 54px;
+        top: 150px;
+        color: #000000;
+        background: #ffffff;
+        font-size: 34px;
+        font-weight: 700;
+        padding: 10px 18px;
+      }
+
+      .block {
+        position: absolute;
+        left: 56px;
+        top: 220px;
+        width: 360px;
+        height: 92px;
+        background: linear-gradient(
+          90deg,
+          #000000 0 25%,
+          #ffffff 25% 50%,
+          #000000 50% 75%,
+          #ffffff 75% 100%
+        );
+        border: 8px solid #000000;
+      }
+
+      .bottom {
+        position: absolute;
+        left: 48px;
+        top: 1370px;
+        color: #ffffff;
+        font-size: 60px;
+        font-weight: 800;
+        text-shadow: 5px 5px 0 #000000;
+      }
+    </style>
+    <script>
+      setTimeout(() => {
+        document.body.classList.add("ready");
+      }, 1500);
+    </script>
+  </head>
+  <body>
+    <div class="content">
+      <div class="label">PASEO CAPTURE HARNESS</div>
+      <div class="sub">DELAYED FIRST PAINT TARGET</div>
+      <div class="block"></div>
+      <div class="bottom">FULL PAGE MARKER AT Y=1370</div>
+    </div>
+  </body>
+</html>

packages/desktop/capture-harness/main.js

@@ -9,6 +9,7 @@ const VIEWPORT_WIDTH = 1280;
 const VIEWPORT_HEIGHT = 800;
 const FULL_PAGE_HEIGHT = 1600;
 const CAPTURE_TIMEOUT_MS = 5000;
+const CAPTURE_RETRY_INTERVAL_MS = 200;
 const REPEAT_COUNT = 5;
 
 function fileUrl(filePath) {
@@ -143,6 +144,10 @@ async function saveImage(image, outputPath) {
   await fsp.writeFile(outputPath, image.toPNG());
 }
 
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 async function waitForGuestLoad(contents, input = {}) {
   await new Promise((resolve) => {
     if (!contents.isLoading()) {
@@ -240,6 +245,7 @@ async function captureWithPrep({
   guestMetrics,
   repeatTotal = REPEAT_COUNT,
   label = "prep",
+  retryUntilPass = false,
 }) {
   const preparation = await renderer(
     win,
@@ -250,26 +256,43 @@ async function captureWithPrep({
     `${mode}-webview-${targetIndex + 1}-${label}-${repeatIndex}.png`,
   );
   try {
-    const image =
-      mode === "viewport" ? await capturePageSequence(contents) : await captureFullPage(contents);
-    await saveImage(image, outputPath);
     const expected =
       mode === "viewport"
         ? { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 }
         : { width: VIEWPORT_WIDTH, height: FULL_PAGE_HEIGHT, minBrightRatio: 0.55 };
-    const analysis = analyzeImage(image, expected, guestMetrics);
+    const deadline = Date.now() + CAPTURE_TIMEOUT_MS;
-    const size = `${analysis.width}x${analysis.height}`;
+    let attempt = 0;
-    const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`;
+    let lastFailure = "capture did not run";
-    const bright = analysis.brightRatio.toFixed(4);
+    while (Date.now() < deadline) {
-    if (!analysis.pass) {
+      attempt += 1;
-      fail(
+      try {
-        `${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
+        const image =
-      );
+          mode === "viewport"
+            ? await capturePageSequence(contents)
+            : await captureFullPage(contents);
+        const analysis = analyzeImage(image, expected, guestMetrics);
+        const size = `${analysis.width}x${analysis.height}`;
+        const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`;
+        const bright = analysis.brightRatio.toFixed(4);
+        if (analysis.pass) {
+          await saveImage(image, outputPath);
+          pass(
+            `${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} attempts=${attempt} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
+          );
+          return analysis;
+        }
+        lastFailure = `size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform}`;
+      } catch (error) {
+        lastFailure = error instanceof Error ? error.message : String(error);
+      }
+      if (!retryUntilPass) {
+        break;
+      }
+      await delay(Math.min(CAPTURE_RETRY_INTERVAL_MS, Math.max(0, deadline - Date.now())));
     }
-    pass(
+    fail(
-      `${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
+      `${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} failed attempts=${attempt} last=${lastFailure} file=${outputPath}`,
     );
-    return analysis;
   } finally {
     const restoredState = await renderer(
       win,
@@ -320,14 +343,13 @@ async function expectLegacySecondWebviewFailure({ win, contents, mode, guestMetr
   }
 }
 
-async function captureFreshWebviewImmediately({ win, waitForNextAttachedGuest }) {
+async function captureFreshDelayedWebview({ win, waitForNextAttachedGuest, mode }) {
   const freshGuestPromise = waitForNextAttachedGuest();
   const targetIndex = await renderer(
     win,
-    `window.captureHarness.addWebview(${JSON.stringify(fileUrl(path.join(ROOT, "bright.html")))})`,
+    `window.captureHarness.addWebview(${JSON.stringify(fileUrl(path.join(ROOT, "delayed-bright.html")))})`,
   );
   const guest = await withTimeout(freshGuestPromise, "fresh did-attach-webview");
-  await waitForGuestLoad(guest, { settleMs: 0 });
   const guestMetrics = await readGuestMetrics(guest);
   if (guestMetrics.innerWidth !== VIEWPORT_WIDTH || guestMetrics.innerHeight !== VIEWPORT_HEIGHT) {
     fail(
@@ -340,22 +362,13 @@ async function captureFreshWebviewImmediately({ win, waitForNextAttachedGuest })
   await captureWithPrep({
     win,
     contents: guest,
-    mode: "viewport",
+    mode,
     repeatIndex: 1,
     targetIndex,
     guestMetrics,
     repeatTotal: 1,
-    label: "fresh-immediate",
+    label: "fresh-delayed-first-frame",
-  });
+    retryUntilPass: true,
-  await captureWithPrep({
-    win,
-    contents: guest,
-    mode: "full-page",
-    repeatIndex: 1,
-    targetIndex,
-    guestMetrics,
-    repeatTotal: 1,
-    label: "fresh-immediate",
   });
 }
 
@@ -458,7 +471,9 @@ async function main() {
 
   await renderer(win, "window.captureHarness.restoreParking()");
 
-  await captureFreshWebviewImmediately({ win, waitForNextAttachedGuest });
+  await captureFreshDelayedWebview({ win, waitForNextAttachedGuest, mode: "viewport" });
+  await renderer(win, "window.captureHarness.restoreParking()");
+  await captureFreshDelayedWebview({ win, waitForNextAttachedGuest, mode: "full-page" });
   await renderer(win, "window.captureHarness.restoreParking()");
 
   const results = [];

packages/desktop/src/features/browser-automation/service.test.ts

@@ -54,11 +54,13 @@ class FakeTab implements TabContents {
   public captureNeverPaints = false;
   public captureThrows = false;
   public captureErrorMessage = "capture failed";
+  public viewportCaptureFailuresBeforeSuccess = 0;
   public deferCaptures = false;
   public prepareNeverAcks = false;
   public prepareErrorMessage: string | null = null;
   public fullPageScreenshotThrows = false;
   public fullPageScreenshotErrorMessage = "UnknownVizError";
+  public fullPageCaptureFailuresBeforeSuccess = 0;
   public layoutMetrics = {
     cssLayoutViewport: { clientWidth: 390, clientHeight: 844 },
     cssContentSize: { width: 390, height: 1200 },
@@ -133,6 +135,10 @@ class FakeTab implements TabContents {
     this.capturedViewports.push(options ?? {});
     this.actions.push("capture");
     this.resolveCaptureStartWaiters();
+    if (this.viewportCaptureFailuresBeforeSuccess > 0) {
+      this.viewportCaptureFailuresBeforeSuccess -= 1;
+      throw new Error(this.captureErrorMessage);
+    }
     if (this.captureThrows) {
       throw new Error(this.captureErrorMessage);
     }
@@ -191,6 +197,10 @@ class FakeTab implements TabContents {
       return this.layoutMetrics;
     }
     if (command === "Page.captureScreenshot") {
+      if (this.fullPageCaptureFailuresBeforeSuccess > 0) {
+        this.fullPageCaptureFailuresBeforeSuccess -= 1;
+        throw new Error(this.fullPageScreenshotErrorMessage);
+      }
       if (this.fullPageScreenshotThrows) {
         throw new Error(this.fullPageScreenshotErrorMessage);
       }
@@ -990,34 +1000,82 @@ describe("executeAutomationCommand", () => {
     ]);
   });
 
-  test("screenshot returns no-frame and restores capture preparation when viewport capture throws UnknownVizError", async () => {
+  test("screenshot retries UnknownVizError until the first viewport frame appears", async () => {
-    const browser = new BrowserAutomationHarness();
+    vi.useFakeTimers();
-    browser.tab.captureThrows = true;
+    try {
-    browser.tab.captureErrorMessage = "UnknownVizError";
+      const browser = new BrowserAutomationHarness();
+      browser.tab.captureErrorMessage = "UnknownVizError";
+      browser.tab.viewportCaptureFailuresBeforeSuccess = 2;
 
-    await expect(
+      const resultPromise = browser.execute({
-      browser.execute({
         command: "screenshot",
         args: { browserId: BROWSER_A },
-      }),
+      });
-    ).resolves.toEqual({
+      await vi.advanceTimersByTimeAsync(400);
-      requestId: "req-screenshot",
-      ok: false,
-      error: {
-        code: "screenshot_no_frame",
-        message: "The browser tab has no painted frame. Focus the tab in the app, then try again.",
-        retryable: false,
-      },
-    });
 
-    expect(browser.tab.actions).toEqual([
+      await expect(resultPromise).resolves.toEqual({
-      "prepare",
+        requestId: "req-screenshot",
-      "background:false",
+        ok: true,
-      "invalidate",
+        result: {
-      "capture",
+          command: "screenshot",
-      "restore:capture-1",
+          browserId: BROWSER_A,
-      "background:true",
+          mimeType: "image/png",
-    ]);
+          dataBase64: "iVBORwECAw==",
+          width: 640,
+          height: 480,
+        },
+      });
+      expect(browser.tab.capturedViewports).toHaveLength(3);
+      expect(browser.tab.actions).toEqual([
+        "prepare",
+        "background:false",
+        "invalidate",
+        "capture",
+        "invalidate",
+        "capture",
+        "invalidate",
+        "capture",
+        "restore:capture-1",
+        "background:true",
+      ]);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  test("screenshot returns no-frame after UnknownVizError spends the retry budget", async () => {
+    vi.useFakeTimers();
+    try {
+      const browser = new BrowserAutomationHarness();
+      browser.tab.captureThrows = true;
+      browser.tab.captureErrorMessage = "UnknownVizError";
+
+      const resultPromise = browser.execute({
+        command: "screenshot",
+        args: { browserId: BROWSER_A },
+      });
+      await vi.advanceTimersByTimeAsync(5_000);
+
+      await expect(resultPromise).resolves.toEqual({
+        requestId: "req-screenshot",
+        ok: false,
+        error: {
+          code: "screenshot_no_frame",
+          message:
+            "The browser tab has no painted frame. Focus the tab in the app, then try again.",
+          retryable: false,
+        },
+      });
+      expect(browser.tab.capturedViewports.length).toBeGreaterThan(1);
+      expect(browser.tab.actions.filter((action) => action === "invalidate")).toHaveLength(
+        browser.tab.capturedViewports.length,
+      );
+      expect(browser.tab.restoredPixelCaptureTokens).toEqual(["capture-1"]);
+      expect(browser.tab.actions.at(-2)).toBe("restore:capture-1");
+      expect(browser.tab.actions.at(-1)).toBe("background:true");
+    } finally {
+      vi.useRealTimers();
+    }
   });
 
   test("screenshot returns no-frame when capture preparation does not ack", async () => {
@@ -1256,33 +1314,45 @@ describe("executeAutomationCommand", () => {
     ]);
   });
 
-  test("screenshot with fullPage returns no-frame when CDP capture throws", async () => {
+  test("screenshot with fullPage retries UnknownVizError until the first CDP frame appears", async () => {
-    const browser = new BrowserAutomationHarness();
+    vi.useFakeTimers();
-    browser.tab.fullPageScreenshotThrows = true;
+    try {
+      const browser = new BrowserAutomationHarness();
+      browser.tab.fullPageCaptureFailuresBeforeSuccess = 1;
 
-    const result = await browser.execute({
+      const resultPromise = browser.execute({
-      command: "screenshot",
+        command: "screenshot",
-      args: { browserId: BROWSER_A, fullPage: true },
+        args: { browserId: BROWSER_A, fullPage: true },
-    });
+      });
+      await vi.advanceTimersByTimeAsync(200);
 
-    expect(result).toEqual({
+      await expect(resultPromise).resolves.toEqual({
-      requestId: "req-screenshot",
+        requestId: "req-screenshot",
-      ok: false,
+        ok: true,
-      error: {
+        result: {
-        code: "screenshot_no_frame",
+          command: "screenshot",
-        message: "The browser tab has no painted frame. Focus the tab in the app, then try again.",
+          browserId: BROWSER_A,
-        retryable: false,
+          mimeType: "image/png",
-      },
+          dataBase64: "fullPagePng",
-    });
+          width: 390,
-    expect(browser.tab.actions).toEqual([
+          height: 1200,
-      "prepare",
+        },
-      "background:false",
+      });
-      "invalidate",
+      expect(browser.tab.actions).toEqual([
-      "debug:Page.getLayoutMetrics",
+        "prepare",
-      "debug:Page.captureScreenshot",
+        "background:false",
-      "restore:capture-1",
+        "invalidate",
-      "background:true",
+        "debug:Page.getLayoutMetrics",
-    ]);
+        "debug:Page.captureScreenshot",
+        "invalidate",
+        "debug:Page.getLayoutMetrics",
+        "debug:Page.captureScreenshot",
+        "restore:capture-1",
+        "background:true",
+      ]);
+    } finally {
+      vi.useRealTimers();
+    }
   });
 
   test("screenshot with fullPage restores capture preparation when CDP capture fails with an ordinary error", async () => {

packages/desktop/src/features/browser-automation/service.ts

@@ -61,6 +61,7 @@ const defaultSnapshotEngine = new BrowserSnapshotEngine();
 const DEFAULT_WAIT_TIMEOUT_MS = 5_000;
 const WAIT_POLL_INTERVAL_MS = 25;
 const PIXEL_CAPTURE_TIMEOUT_MS = 5_000;
+const PIXEL_CAPTURE_RETRY_INTERVAL_MS = 200;
 const SCREENSHOT_NO_FRAME_MESSAGE =
   "The browser tab has no painted frame. Focus the tab in the app, then try again.";
 const SCREENSHOT_PREP_UNAVAILABLE_PREFIX = "Browser screenshot prep_unavailable:";
@@ -94,12 +95,18 @@ function screenshotNoFrameFailure(
   return fail(requestId, "screenshot_no_frame", error.message);
 }
 
-async function withPixelCaptureTimeout<T>(capture: Promise<T>): Promise<T> {
+async function withPixelCaptureTimeout<T>(
+  capture: Promise<T>,
+  timeoutMs = PIXEL_CAPTURE_TIMEOUT_MS,
+): Promise<T> {
+  if (timeoutMs <= 0) {
+    throw new ScreenshotNoFrameError();
+  }
   let timeoutId: ReturnType<typeof setTimeout> | undefined;
   const timeout = new Promise<never>((_, reject) => {
     timeoutId = setTimeout(() => {
       reject(new ScreenshotNoFrameError());
-    }, PIXEL_CAPTURE_TIMEOUT_MS);
+    }, timeoutMs);
   });
 
   try {
@@ -150,18 +157,26 @@ async function restorePixelCapture(
   }
 }
 
-async function capturePreparedPixelFrame<T>(capture: () => Promise<T>): Promise<T> {
+async function capturePreparedPixelFrame<T>(
-  try {
+  contents: TabContents,
-    return await withPixelCaptureTimeout(capture());
+  capture: () => Promise<T>,
-  } catch (error) {
+): Promise<T> {
-    if (isScreenshotNoFrameError(error)) {
+  const deadline = Date.now() + PIXEL_CAPTURE_TIMEOUT_MS;
-      throw error;
+  while (Date.now() < deadline) {
+    try {
+      contents.invalidate();
+      return await withPixelCaptureTimeout(capture(), deadline - Date.now());
+    } catch (error) {
+      if (isScreenshotNoFrameError(error)) {
+        throw error;
+      }
+      if (!isKnownNoFrameCaptureError(error)) {
+        throw error;
+      }
+      await delay(Math.min(PIXEL_CAPTURE_RETRY_INTERVAL_MS, Math.max(0, deadline - Date.now())));
     }
-    if (isKnownNoFrameCaptureError(error)) {
-      throw new ScreenshotNoFrameError();
-    }
-    throw error;
   }
+  throw new ScreenshotNoFrameError();
 }
 
 function screenshotPreparationError(error: unknown): ScreenshotNoFrameError {
@@ -206,8 +221,7 @@ async function runPreparedPixelCapture<T>(
       // paintable before capture; see docs/browser-capture-harness.md.
       preparation = await prepareForPixelCapture(contents);
       contents.setBackgroundThrottling(false);
-      contents.invalidate();
+      return await capturePreparedPixelFrame(contents, capture);
-      return await capturePreparedPixelFrame(capture);
     } finally {
       try {
         if (preparation) {