Keep desktop browser tabs stable across focus and automation (#2907)

* fix(desktop): preserve browser tabs across focus changes Electron can replace a guest WebContents when a webview is reparented, which reloads the page and can race automation against a destroyed target. Keep each browser guest on one permanent paintable surface and make user and agent resizing share one canonical viewport. * fix(desktop): recover browser surfaces after lifecycle changes * fix(desktop): preserve resident browser lifecycle state * fix(desktop): seed inactive browser viewport
2026-08-06 00:05:21 +02:00
parent 7f7e8923eb
commit eb95a6eea6
12 changed files with 929 additions and 84 deletions
@@ -272,11 +272,68 @@ async function readGuest(page, browserId) {
    return {
      webContentsId: webview.getWebContentsId(),
      parentId: webview.parentElement?.id ?? null,
+      width: Math.round(webview.getBoundingClientRect().width),
+      height: Math.round(webview.getBoundingClientRect().height),
    };
  }, browserId);
 }

-async function runRegression({ page, client, serverId, targetUrl }) {
+async function readPresentation(page, browserId) {
+  return await page.evaluate((id) => {
+    const surface = document.querySelector(`[data-paseo-browser-surface="${id}"]`);
+    const clip = document.querySelector(`[data-testid="browser-webview-clip-${id}"]`);
+    if (!(surface instanceof HTMLElement) || !(clip instanceof HTMLElement)) return null;
+    const surfaceRect = surface.getBoundingClientRect();
+    const clipRect = clip.getBoundingClientRect();
+    const webview = surface.querySelector(`[data-paseo-browser-id="${id}"]`);
+    if (!(webview instanceof HTMLElement)) return null;
+    const webviewRect = webview.getBoundingClientRect();
+    const outsidePoint = {
+      x: Math.max(0, Math.round(clipRect.left - 1)),
+      y: Math.round(clipRect.top + clipRect.height / 2),
+    };
+    const outsideTarget = document.elementFromPoint(outsidePoint.x, outsidePoint.y);
+    return {
+      surface: {
+        left: Math.round(surfaceRect.left),
+        top: Math.round(surfaceRect.top),
+        right: Math.round(surfaceRect.right),
+        bottom: Math.round(surfaceRect.bottom),
+      },
+      clip: {
+        left: Math.round(clipRect.left),
+        top: Math.round(clipRect.top),
+        right: Math.round(clipRect.right),
+        bottom: Math.round(clipRect.bottom),
+      },
+      webview: {
+        left: Math.round(webviewRect.left),
+        top: Math.round(webviewRect.top),
+      },
+      capturesOutsideInput: surface.contains(outsideTarget),
+    };
+  }, browserId);
+}
+
+async function readViewport(client, browserId) {
+  const evaluated = await callBrowserTool(client, "browser_evaluate", {
+    browserId,
+    function: "() => ({ width: window.innerWidth, height: window.innerHeight })",
+  });
+  return JSON.parse(evaluated.resultJson);
+}
+
+function recordViewportMismatch(failures, label, actual, expected) {
+  if (actual.width === expected.width && actual.height === expected.height) {
+    return;
+  }
+  failures.push(
+    `${label}: expected ${expected.width}x${expected.height}, received ${actual.width}x${actual.height}`,
+  );
+}
+
+async function runRegression({ page, client, serverId, targetUrl, callerAgentId }) {
+  const failures = [];
  const originalWorkspaceId = workspaceIds[0];
  const originalWorkspaceRow = page.getByTestId(
    `sidebar-workspace-row-${serverId}:${originalWorkspaceId}`,
@@ -300,6 +357,117 @@ async function runRegression({ page, client, serverId, targetUrl }) {
  );
  const firstGuest = await readGuest(page, browserId);
  assert(firstGuest, "Original browser guest was not attached to its workspace pane");
+  recordViewportMismatch(
+    failures,
+    "Responsive viewport follows the visible browser pane",
+    await readViewport(client, browserId),
+    { width: firstGuest.width, height: firstGuest.height },
+  );
+
+  await callBrowserTool(client, "browser_wait", {
+    browserId,
+    text: "Bridge target",
+    timeoutMs: 5_000,
+  });
+  const requestedViewport = { width: 640, height: 480 };
+  await callBrowserTool(client, "browser_resize", { browserId, ...requestedViewport });
+  recordViewportMismatch(
+    failures,
+    "browser_resize updates the visible shared viewport",
+    await readViewport(client, browserId),
+    requestedViewport,
+  );
+
+  const oversizedViewport = { width: 2560, height: 1440 };
+  await callBrowserTool(client, "browser_resize", { browserId, ...oversizedViewport });
+  recordViewportMismatch(
+    failures,
+    "oversized preset preserves the requested guest viewport",
+    await readViewport(client, browserId),
+    oversizedViewport,
+  );
+  await page.waitForFunction(
+    ({ id, width, height }) => {
+      const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
+      return (
+        webview instanceof HTMLElement &&
+        Math.round(webview.getBoundingClientRect().width) === width &&
+        Math.round(webview.getBoundingClientRect().height) === height
+      );
+    },
+    { id: browserId, ...oversizedViewport },
+    { timeout: timeoutMs },
+  );
+  const presentation = await readPresentation(page, browserId);
+  assert(presentation, "Oversized browser presentation geometry was unavailable");
+  if (
+    presentation.surface.left < presentation.clip.left ||
+    presentation.surface.top < presentation.clip.top ||
+    presentation.surface.right > presentation.clip.right ||
+    presentation.surface.bottom > presentation.clip.bottom
+  ) {
+    failures.push(
+      `oversized browser surface stays inside its pane: ${JSON.stringify(presentation)}`,
+    );
+  }
+  if (presentation.capturesOutsideInput) {
+    failures.push("oversized browser surface captures input outside its pane");
+  }
+  await callBrowserTool(client, "browser_resize", { browserId, ...oversizedViewport });
+  const repeatedResizePresentation = await readPresentation(page, browserId);
+  assert(repeatedResizePresentation, "Repeated resize presentation geometry was unavailable");
+  if (
+    repeatedResizePresentation.webview.left !== presentation.webview.left ||
+    repeatedResizePresentation.webview.top !== presentation.webview.top
+  ) {
+    failures.push(
+      `repeating an oversized resize preserves the guest offset: before ${JSON.stringify(presentation.webview)}, after ${JSON.stringify(repeatedResizePresentation.webview)}`,
+    );
+  }
+  await callBrowserTool(client, "browser_resize", { browserId, ...requestedViewport });
+
+  await originalDeck.getByTestId(`workspace-tab-agent_${callerAgentId}`).click();
+  await page.waitForTimeout(500);
+  try {
+    await callBrowserTool(client, "browser_screenshot", { browserId });
+  } catch (error) {
+    failures.push(`inactive browser remains captureable: ${String(error)}`);
+  }
+  recordViewportMismatch(
+    failures,
+    "inactive browser preserves the shared viewport",
+    await readViewport(client, browserId),
+    requestedViewport,
+  );
+  const focusContinuitySentinel = "preserve-browser-document-across-focus";
+  await callBrowserTool(client, "browser_evaluate", {
+    browserId,
+    function: `() => { globalThis.__paseoFocusContinuity = ${JSON.stringify(focusContinuitySentinel)}; return globalThis.__paseoFocusContinuity; }`,
+  });
+
+  await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).click();
+  await page.waitForFunction(
+    ({ id, webContentsId }) => {
+      const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
+      return (
+        webview?.parentElement?.getAttribute("data-paseo-browser-surface") === id &&
+        webview.parentElement.style.pointerEvents === "auto" &&
+        webview.getWebContentsId() === webContentsId
+      );
+    },
+    { id: browserId, webContentsId: firstGuest.webContentsId },
+    { timeout: timeoutMs },
+  );
+  const continuityResult = await callBrowserTool(client, "browser_evaluate", {
+    browserId,
+    function: "() => globalThis.__paseoFocusContinuity ?? null",
+  });
+  const continuityValue = JSON.parse(continuityResult.resultJson);
+  if (continuityValue !== focusContinuitySentinel) {
+    failures.push(
+      `focusing the browser tab preserves the current document: expected ${JSON.stringify(focusContinuitySentinel)}, received ${JSON.stringify(continuityValue)}`,
+    );
+  }

  for (const workspaceId of workspaceIds.slice(1)) {
    await page.getByTestId(`sidebar-workspace-row-${serverId}:${workspaceId}`).click();
@@ -312,21 +480,22 @@ async function runRegression({ page, client, serverId, targetUrl }) {
    ({ id, previousWebContentsId }) => {
      const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
      return (
-        webview?.parentElement?.id === "paseo-browser-resident-webviews" &&
+        webview?.parentElement?.getAttribute("data-paseo-browser-surface") === id &&
+        webview.parentElement.style.width === "1px" &&
        typeof webview.getWebContentsId === "function" &&
-        webview.getWebContentsId() !== previousWebContentsId
+        webview.getWebContentsId() === previousWebContentsId
      );
    },
    { id: browserId, previousWebContentsId: firstGuest.webContentsId },
    { timeout: timeoutMs },
  );
-  const replacementGuest = await readGuest(page, browserId);
-  assert(replacementGuest, "Replacement browser guest was not parked after workspace eviction");
+  const parkedGuest = await readGuest(page, browserId);
+  assert(parkedGuest, "Browser guest was not parked after workspace eviction");

  const listed = await callBrowserTool(client, "browser_list_tabs");
  assert(
    listed.tabs.some((tab) => tab.browserId === browserId),
-    "browser_list_tabs lost the original tab after guest replacement",
+    "browser_list_tabs lost the original tab after workspace eviction",
  );

  const snapshot = await callBrowserTool(client, "browser_snapshot", { browserId });
@@ -344,10 +513,47 @@ async function runRegression({ page, client, serverId, targetUrl }) {
    timeoutMs: 5_000,
  });

+  await originalWorkspaceRow.click();
+  await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).click();
+  await page.waitForFunction(
+    ({ id, webContentsId }) => {
+      const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
+      return (
+        webview?.parentElement?.getAttribute("data-paseo-browser-surface") === id &&
+        webview.parentElement.style.pointerEvents === "auto" &&
+        webview.getWebContentsId() === webContentsId
+      );
+    },
+    { id: browserId, webContentsId: firstGuest.webContentsId },
+    { timeout: timeoutMs },
+  );
+  recordViewportMismatch(
+    failures,
+    "browser viewport survives workspace eviction and reattachment",
+    await readViewport(client, browserId),
+    requestedViewport,
+  );
+
+  await originalDeck.getByRole("button", { name: "Annotate element" }).click();
+  await page.waitForTimeout(250);
+  const selectorResult = await callBrowserTool(client, "browser_evaluate", {
+    browserId,
+    function: "() => Boolean(globalThis.__paseoSelector)",
+  });
+  if (JSON.parse(selectorResult.resultJson) !== true) {
+    failures.push("reused loaded browser remains ready for element annotation after remount");
+  }
+
+  if (failures.length > 0) {
+    throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`);
+  }
+
  return {
    browserId,
    originalWebContentsId: firstGuest.webContentsId,
-    replacementWebContentsId: replacementGuest.webContentsId,
+    finalWebContentsId: parkedGuest.webContentsId,
+    viewport: "passed",
+    inactiveCapture: "passed",
    list: "passed",
    snapshot: "passed",
    click: "passed",
@@ -446,10 +652,11 @@ async function main() {
      client,
      serverId: status.serverId,
      targetUrl: target.url,
+      callerAgentId,
    });
    writeJson(path.join(artifactDir, "result.json"), report);
    console.log(
-      `Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} -> ${report.replacementWebContentsId}; list, snapshot, click passed.`,
+      `Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click passed.`,
    );
  } catch (error) {
    console.error(`Browser desktop browser E2E failed. Artifacts: ${artifactDir}`);
@@ -20,11 +20,14 @@ class FakeDebugger {
  public blockCommands = false;
  public readonly blockedCommandNames = new Set<string>();
  public readonly failedCommandNames = new Set<string>();
+  public readonly failedCommandErrors = new Map<string, Error>();
  public readonly promptDialogs: unknown[] = [];
  public failPromptDrain = false;
+  public beforeCommand: ((command: string) => void) | null = null;
  private messageListener:
    | ((event: unknown, method: string, params?: Record<string, unknown>) => void)
    | null = null;
+  private detachListener: (() => void) | null = null;
  private readonly blockedCommands: Array<() => void> = [];

  public isAttached(): boolean {
@@ -37,6 +40,11 @@ class FakeDebugger {

  public async sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown> {
    this.commands.push({ command, params: params ?? {} });
+    this.beforeCommand?.(command);
+    const commandError = this.failedCommandErrors.get(command);
+    if (commandError) {
+      throw commandError;
+    }
    if (this.failedCommandNames.has(command)) {
      throw new Error(`${command} failed`);
    }
@@ -58,10 +66,15 @@ class FakeDebugger {
  }

  public on(
-    event: "message",
-    listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
+    event: "message" | "detach",
+    listener:
+      | ((event: unknown, method: string, params?: Record<string, unknown>) => void)
+      | (() => void),
  ): void {
-    expect(event).toBe("message");
+    if (event === "detach") {
+      this.detachListener = listener;
+      return;
+    }
    this.messageListener = listener;
  }

@@ -72,6 +85,11 @@ class FakeDebugger {
    this.messageListener({}, method, params);
  }

+  public emitDetach(): void {
+    this.attachedProtocolVersions.length = 0;
+    this.detachListener?.();
+  }
+
  public finishNextCommand(): void {
    const resolve = this.blockedCommands.shift();
    if (!resolve) {
@@ -92,6 +110,7 @@ type ConsoleMessageListener = (
 class FakeWebContents {
  public readonly debugger = new FakeDebugger();
  public readonly inputEvents: IsolatedKeyboardInputEvent[] = [];
+  public readonly loadedUrls: string[] = [];
  public readonly captures: Array<{
    rect: Rectangle | undefined;
    options: { stayHidden?: boolean } | undefined;
@@ -138,7 +157,9 @@ class FakeWebContents {
    return null;
  }

-  public async loadURL(): Promise<void> {}
+  public async loadURL(url: string): Promise<void> {
+    this.loadedUrls.push(url);
+  }

  public goBack(): void {}

@@ -186,6 +207,7 @@ class FakeWebContents {

  public destroy(): void {
    this.destroyed = true;
+    this.debugger.emitDetach();
    this.destroyedListener?.();
  }
 }
@@ -571,6 +593,54 @@ describe("browser automation IPC adapter", () => {
    );
    warn.mockRestore();
  });
+
+  test("does not run the command after the debugger target closes during setup", async () => {
+    const contents = new FakeWebContents(31);
+    contents.debugger.beforeCommand = (command) => {
+      if (command === "Page.enable") {
+        contents.destroy();
+      }
+    };
+    contents.debugger.failedCommandErrors.set(
+      "Page.enable",
+      new Error("target closed while handling command"),
+    );
+    const tab = adaptWebContents(contents);
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+    await expect(
+      tab.captureDialogs?.(() => tab.loadURL("https://replacement.example.com")),
+    ).rejects.toThrow("target closed while handling command");
+    expect(contents.loadedUrls).toEqual([]);
+    expect(warn).not.toHaveBeenCalled();
+
+    warn.mockRestore();
+  });
+
+  test("runs without dialog capture after a live debugger session detaches", async () => {
+    const contents = new FakeWebContents(32);
+    const tab = adaptWebContents(contents);
+
+    await expect(tab.captureDialogs?.(async () => "captured")).resolves.toEqual({
+      result: "captured",
+      dialogs: [],
+    });
+    contents.debugger.emitDetach();
+    contents.debugger.failedCommandNames.add("Page.enable");
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+    await expect(
+      tab.captureDialogs?.(() => tab.loadURL("https://replacement.example.com")),
+    ).resolves.toEqual({ result: undefined, dialogs: [] });
+    expect(contents.loadedUrls).toEqual(["https://replacement.example.com"]);
+    expect(contents.debugger.attachedProtocolVersions).toEqual(["1.3"]);
+    expect(warn).toHaveBeenCalledWith(
+      "[browser-automation] Dialog capture unavailable; running command without it",
+      { contentsId: 32, error: expect.any(Error) },
+    );
+
+    warn.mockRestore();
+  });
 });

 class FakeHostWebContents {
@@ -74,6 +74,7 @@ interface WebContentsDebugger {
    event: "message",
    listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
  ): void;
+  on?(event: "detach", listener: () => void): void;
 }

 interface ConsoleMessageEmitter {
@@ -188,6 +189,7 @@ function getDialogMonitor(
 class DialogMonitor {
  private enabled = false;
  private listenerRegistered = false;
+  private detachGeneration = 0;
  private readonly activeCollectors: DialogCollector[] = [];

  public constructor(
@@ -200,10 +202,14 @@ class DialogMonitor {
    task: () => Promise<T>,
  ): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }> {
    const collector: DialogCollector = { dialogs: [] };
+    const setupDetachGeneration = this.detachGeneration;
    try {
      await this.enable();
      await this.installPromptShim();
    } catch (error) {
+      if (this.contents.isDestroyed() || this.detachGeneration !== setupDetachGeneration) {
+        throw error;
+      }
      console.warn("[browser-automation] Dialog capture unavailable; running command without it", {
        contentsId: this.contentsId,
        error,
@@ -244,6 +250,10 @@ class DialogMonitor {
        }
        void this.handleOpening(params ?? {});
      });
+      this.contents.debugger.on("detach", () => {
+        this.enabled = false;
+        this.detachGeneration += 1;
+      });
    }
    await this.sendDebugCommand("Page.enable");
    this.enabled = true;