fix(desktop): keep browser surfaces interactive

2026-08-07 22:34:59 +02:00
parent 98d3281cad
commit dc19b45f23
9 changed files with 234 additions and 34 deletions
@@ -87,6 +87,14 @@ or parking changes that surface's geometry without reparenting the webview. The
 uses `left:0`, `top:0`, `width:1px`, `height:1px`, `overflow:hidden`, `opacity:1`, and
 `pointer-events:none`. The webview stays at its resolved logical viewport, defaulting to
 1280x800 before first presentation, with `display:inline-flex` at `left:0`, `top:0`.
+Presentation resolves responsive guests to the pane's exact pixel dimensions after the surface
+has visible bounds. Do not apply percentage guest sizing against the parked surface: Electron
+exposes the 1x1 parking geometry as a real guest resize before expanding it again.
+
+The permanent browser host and `overlay-root` are explicit sibling paint planes. The browser
+plane stays below the overlay plane regardless of body insertion order; menus keep their relative
+layering inside `overlay-root`. Activating a presented browser also focuses its registered guest
+`WebContents` in main so macOS assigns keyboard first-responder ownership to the page.

 There is no renderer prep/restore handshake. Main disables guest background throttling
 once when the webview attaches, then screenshot capture uses the shared serialized queue,
@@ -58,7 +58,6 @@ import {
  useBrowserStore,
 } from "@/desktop/browser/store";
 import {
-  applyBrowserWebviewViewport,
  applyInactiveBrowserWebviewViewport,
  isResidentBrowserWebviewReady,
  prepareBrowserWebview,
@@ -787,8 +786,7 @@ export function BrowserPane({
    }
    releaseResidentBrowserWebview(browserId, webview);
    if (isPresentedRef.current) {
-      applyBrowserWebviewViewport(webview, browserViewportRef.current);
-      presentBrowserWebview(browserId, webview, host, clip);
+      presentBrowserWebview(browserId, webview, host, clip, browserViewportRef.current);
    } else {
      applyInactiveBrowserWebviewViewport(browserId, webview, browserViewportRef.current);
    }
@@ -799,7 +797,13 @@ export function BrowserPane({
            if (!isPresentedRef.current) {
              return;
            }
-            presentBrowserWebview(browserIdRef.current, webview, host, clip);
+            presentBrowserWebview(
+              browserIdRef.current,
+              webview,
+              host,
+              clip,
+              browserViewportRef.current,
+            );
            rememberResolvedBrowserWebviewSize(browserIdRef.current, webview);
          });

@@ -882,6 +886,13 @@ export function BrowserPane({
    };
    const handleWebviewFocus = () => {
      onFocusPane?.();
+      webview.focus?.();
+      const focusBrowser = getDesktopHost()?.browser?.focus;
+      if (typeof focusBrowser === "function") {
+        void focusBrowser(browserIdRef.current).catch((error) => {
+          console.error("[browser-webview] focus failed", error);
+        });
+      }
    };

    webview.addEventListener("did-start-loading", handleStartLoading);
@@ -946,11 +957,10 @@ export function BrowserPane({
      releaseResidentBrowserWebview(browserId, webview);
      return;
    }
-    applyBrowserWebviewViewport(webview, browserViewport);
    const host = webviewHostRef.current;
    const clip = webviewClipRef.current;
    if (host && clip) {
-      presentBrowserWebview(browserId, webview, host, clip);
+      presentBrowserWebview(browserId, webview, host, clip, browserViewport);
    }
    if (browserViewport.mode === "fixed") {
      rememberBrowserWebviewSize({
@@ -1,6 +1,5 @@
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
 import {
-  applyBrowserWebviewViewport,
  applyInactiveBrowserWebviewViewport,
  type BrowserWebviewProfileHost,
  clearResidentBrowserWebviewsForTests,
@@ -141,7 +140,9 @@ describe("resident browser webviews", () => {
    }
    const permanentParent = webview.parentElement;

-    presentBrowserWebview("browser-stable-parent", webview, anchor, clip);
+    presentBrowserWebview("browser-stable-parent", webview, anchor, clip, {
+      mode: "responsive",
+    });
    expect(webview.parentElement).toBe(permanentParent);
    expect(permanentParent.style.left).toBe("40px");
    expect(permanentParent.style.top).toBe("60px");
@@ -149,12 +150,17 @@ describe("resident browser webviews", () => {
    expect(permanentParent.style.height).toBe("480px");
    expect(permanentParent.style.pointerEvents).toBe("auto");
    expect(permanentParent.getAttribute("aria-hidden")).toBe("false");
+    expect(webview.style.flex).toBe("0 0 auto");
+    expect(webview.style.width).toBe("640px");
+    expect(webview.style.height).toBe("480px");

    releaseResidentBrowserWebview("browser-stable-parent", webview);
    expect(webview.parentElement).toBe(permanentParent);
    expectParkedSurface(permanentParent);

-    presentBrowserWebview("browser-stable-parent", webview, anchor, clip);
+    presentBrowserWebview("browser-stable-parent", webview, anchor, clip, {
+      mode: "responsive",
+    });
    expect(webview.parentElement).toBe(permanentParent);
  });

@@ -167,7 +173,6 @@ describe("resident browser webviews", () => {
    if (!webview?.parentElement) {
      throw new Error("Expected resident browser surface");
    }
-    applyBrowserWebviewViewport(webview, { mode: "fixed", width: 2560, height: 1440 });
    const anchor = document.createElement("div");
    const clip = document.createElement("div");
    Object.defineProperty(anchor, "getBoundingClientRect", {
@@ -177,7 +182,11 @@ describe("resident browser webviews", () => {
      value: () => ({ left: 100, top: 150, width: 800, height: 600 }),
    });

-    presentBrowserWebview("browser-oversized", webview, anchor, clip);
+    presentBrowserWebview("browser-oversized", webview, anchor, clip, {
+      mode: "fixed",
+      width: 2560,
+      height: 1440,
+    });

    expect(webview.parentElement.style.left).toBe("100px");
    expect(webview.parentElement.style.top).toBe("150px");
@@ -359,8 +368,20 @@ describe("resident browser webviews", () => {

  it("applies an exact fixed viewport without a flex width override", () => {
    const webview = document.createElement("webview");
+    const anchor = document.createElement("div");
+    const clip = document.createElement("div");
+    Object.defineProperty(anchor, "getBoundingClientRect", {
+      value: () => ({ left: 0, top: 0, width: 640, height: 480 }),
+    });
+    Object.defineProperty(clip, "getBoundingClientRect", {
+      value: () => ({ left: 0, top: 0, width: 640, height: 480 }),
+    });

-    applyBrowserWebviewViewport(webview, { mode: "fixed", width: 640, height: 480 });
+    presentBrowserWebview("browser-fixed", webview, anchor, clip, {
+      mode: "fixed",
+      width: 640,
+      height: 480,
+    });

    expect(webview.style.flex).toBe("0 0 auto");
    expect(webview.style.width).toBe("640px");
@@ -4,6 +4,7 @@ import {
  type DesktopBrowserBridge,
 } from "@/desktop/host";
 import type { BrowserViewport } from "@/desktop/browser/store";
+import { WEB_SURFACE_PLANE } from "@/lib/overlay-root";

 const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
 const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
@@ -108,7 +109,7 @@ function applyResidentHostParkingStyle(host: HTMLElement): void {
  host.style.opacity = "1";
  host.style.pointerEvents = "none";
  host.style.display = "block";
-  host.style.zIndex = "0";
+  host.style.zIndex = String(WEB_SURFACE_PLANE.browser);
  host.style.clipPath = "";
  host.style.visibility = "visible";
  host.style.transform = "";
@@ -220,24 +221,16 @@ export function rememberBrowserWebviewSize(input: {
  return dimensions;
 }

-function applyBrowserWebviewDimensions(webview: HTMLElement, viewport: BrowserViewport): void {
+function applyBrowserWebviewDimensions(
+  webview: HTMLElement,
+  dimensions: { width: number; height: number },
+): void {
  webview.style.display = "flex";
  webview.style.border = "0";
  webview.style.background = "transparent";
-  if (viewport.mode === "responsive") {
-    webview.style.flex = "1";
-    webview.style.width = "100%";
-    webview.style.height = "100%";
-    return;
-  }
  webview.style.flex = "0 0 auto";
-  webview.style.width = `${viewport.width}px`;
-  webview.style.height = `${viewport.height}px`;
-}
-
-export function applyBrowserWebviewViewport(webview: HTMLElement, viewport: BrowserViewport): void {
-  clearResidentWebviewParkingStyle(webview);
-  applyBrowserWebviewDimensions(webview, viewport);
+  webview.style.width = `${Math.max(1, Math.round(dimensions.width))}px`;
+  webview.style.height = `${Math.max(1, Math.round(dimensions.height))}px`;
 }

 export function applyInactiveBrowserWebviewViewport(
@@ -256,6 +249,7 @@ export function presentBrowserWebview(
  webview: HTMLElement,
  anchor: HTMLElement,
  clip: HTMLElement,
+  viewport: BrowserViewport,
 ): void {
  const normalizedBrowserId = trimNonEmpty(browserId);
  if (!normalizedBrowserId) {
@@ -297,6 +291,13 @@ export function presentBrowserWebview(
  surface.style.pointerEvents = hasVisibleArea ? "auto" : "none";
  surface.style.display = "flex";
  surface.style.visibility = "visible";
+  clearResidentWebviewParkingStyle(webview);
+  applyBrowserWebviewDimensions(
+    webview,
+    viewport.mode === "responsive"
+      ? { width: anchorBounds.width, height: anchorBounds.height }
+      : viewport,
+  );
  webview.style.position = "absolute";
  webview.style.left = `${Math.round(anchorBounds.left - surfaceLeft)}px`;
  webview.style.top = `${Math.round(anchorBounds.top - surfaceTop)}px`;
@@ -432,7 +433,7 @@ export function resizeResidentBrowserWebview(input: {
  const ownerDocument = readDocument();
  const webview = ownerDocument ? findBrowserWebview(normalizedBrowserId, ownerDocument) : null;
  if (webview) {
-    applyBrowserWebviewDimensions(webview, { mode: "fixed", ...dimensions });
+    applyBrowserWebviewDimensions(webview, dimensions);
  }

  return dimensions;
@@ -151,6 +151,7 @@ export interface DesktopBrowserBridge {
    workspaceId: string;
    browserId: string | null;
  }) => Promise<void>;
+  focus?: (browserId: string) => Promise<boolean>;
  openDevTools?: (browserId: string) => Promise<unknown>;
  clearProfile?: (legacyBrowserIds: string[]) => Promise<void>;
  executeAutomationCommand?: (
@@ -28,14 +28,20 @@ export function getOverlayRoot(): HTMLElement {
  if (!el) {
    el = document.createElement("div");
    el.id = "overlay-root";
-    el.style.position = "fixed";
-    el.style.inset = "0";
-    el.style.pointerEvents = "none";
    document.body.appendChild(el);
  }
+  el.style.position = "fixed";
+  el.style.inset = "0";
+  el.style.pointerEvents = "none";
+  el.style.zIndex = String(WEB_SURFACE_PLANE.overlay);
  return el;
 }

+export const WEB_SURFACE_PLANE = {
+  browser: 0,
+  overlay: 1,
+} as const;
+
 export const OVERLAY_Z = {
  floating: 10,
  modal: 20,
@@ -199,6 +199,8 @@ async function startTargetPage() {
        <head><title>Desktop browser target</title></head>
        <body>
          <button id="bridge-target" onclick="this.textContent = 'Clicked'">Bridge target</button>
+          <label for="typing-target">Typing target</label>
+          <input id="typing-target" />
        </body>
      </html>`);
  });
@@ -323,6 +325,55 @@ async function readViewport(client, browserId) {
  return JSON.parse(evaluated.resultJson);
 }

+async function clickGuestElement(page, client, browserId, selector) {
+  const evaluated = await callBrowserTool(client, "browser_evaluate", {
+    browserId,
+    function: `() => {
+      const element = document.querySelector(${JSON.stringify(selector)});
+      if (!(element instanceof HTMLElement)) return null;
+      const rect = element.getBoundingClientRect();
+      return { x: rect.x, y: rect.y, width: rect.width, height: rect.height };
+    }`,
+  });
+  const elementRect = JSON.parse(evaluated.resultJson);
+  assert(elementRect, `Guest element ${selector} was unavailable`);
+  const webviewRect = await page.evaluate((id) => {
+    const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
+    if (!(webview instanceof HTMLElement)) return null;
+    const rect = webview.getBoundingClientRect();
+    return { x: rect.x, y: rect.y };
+  }, browserId);
+  assert(webviewRect, `Browser webview ${browserId} was unavailable`);
+  await page.mouse.click(
+    webviewRect.x + elementRect.x + elementRect.width / 2,
+    webviewRect.y + elementRect.y + elementRect.height / 2,
+  );
+}
+
+async function selectDeviceSize(page, label) {
+  await page.locator('[aria-label="Device size"]').click();
+  const item = page.getByText(label, { exact: true });
+  await item.waitFor({ state: "visible", timeout: timeoutMs });
+  const rect = await item.boundingBox();
+  assert(rect, `Device size menu item ${label} had no bounds`);
+  const clip = {
+    x: Math.max(0, rect.x),
+    y: Math.max(0, rect.y),
+    width: rect.width,
+    height: rect.height,
+  };
+  const openPixels = await page.screenshot({ clip });
+  await page.keyboard.press("Escape");
+  await item.waitFor({ state: "hidden", timeout: timeoutMs });
+  const closedPixels = await page.screenshot({ clip });
+
+  await page.locator('[aria-label="Device size"]').click();
+  await item.waitFor({ state: "visible", timeout: timeoutMs });
+  await page.mouse.click(rect.x + rect.width / 2, rect.y + rect.height / 2);
+  await page.keyboard.press("Escape");
+  return !openPixels.equals(closedPixels);
+}
+
 function recordViewportMismatch(failures, label, actual, expected) {
  if (actual.width === expected.width && actual.height === expected.height) {
    return;
@@ -341,6 +392,16 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId
  await originalWorkspaceRow.waitFor({ state: "visible", timeout: timeoutMs });
  await originalWorkspaceRow.click();

+  await page.evaluate(() => {
+    if (document.getElementById("overlay-root")) return;
+    const overlayRoot = document.createElement("div");
+    overlayRoot.id = "overlay-root";
+    overlayRoot.style.position = "fixed";
+    overlayRoot.style.inset = "0";
+    overlayRoot.style.pointerEvents = "none";
+    document.body.appendChild(overlayRoot);
+  });
+
  const created = await callBrowserTool(client, "browser_new_tab", { url: targetUrl });
  const browserId = created.browserId;
  assert(typeof browserId === "string", "browser_new_tab returned no browserId");
@@ -364,6 +425,30 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId
    { width: firstGuest.width, height: firstGuest.height },
  );

+  await clickGuestElement(page, client, browserId, "#typing-target");
+  const activeGuestElement = await callBrowserTool(client, "browser_evaluate", {
+    browserId,
+    function: "() => document.activeElement?.id ?? null",
+  });
+  assert(
+    JSON.parse(activeGuestElement.resultJson) === "typing-target",
+    "Physical browser click did not focus the guest input",
+  );
+  const focusedGuest = await page.evaluate(
+    (id) => window.paseoDesktop?.browser?.focus?.(id),
+    browserId,
+  );
+  assert(focusedGuest === true, "Electron did not focus the registered browser guest");
+
+  const deviceSizeMenuPainted = await selectDeviceSize(page, "iPhone SE · 375×667");
+  assert(deviceSizeMenuPainted, "Device size menu did not paint above the browser surface");
+  recordViewportMismatch(
+    failures,
+    "device size menu paints and receives input above the browser surface",
+    await readViewport(client, browserId),
+    { width: 375, height: 667 },
+  );
+
  await callBrowserTool(client, "browser_wait", {
    browserId,
    text: "Bridge target",
@@ -426,6 +511,9 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId
  }
  await callBrowserTool(client, "browser_resize", { browserId, ...requestedViewport });

+  await selectDeviceSize(page, "Responsive");
+  const responsiveViewport = await readViewport(client, browserId);
+
  await originalDeck.getByTestId(`workspace-tab-agent_${callerAgentId}`).click();
  await page.waitForTimeout(500);
  try {
@@ -437,13 +525,40 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId
    failures,
    "inactive browser preserves the shared viewport",
    await readViewport(client, browserId),
-    requestedViewport,
+    responsiveViewport,
  );
  const focusContinuitySentinel = "preserve-browser-document-across-focus";
  await callBrowserTool(client, "browser_evaluate", {
    browserId,
-    function: `() => { globalThis.__paseoFocusContinuity = ${JSON.stringify(focusContinuitySentinel)}; return globalThis.__paseoFocusContinuity; }`,
+    function: `() => {
+      globalThis.__paseoFocusContinuity = ${JSON.stringify(focusContinuitySentinel)};
+      globalThis.__paseoViewportTransitions = [{ width: innerWidth, height: innerHeight }];
+      addEventListener('resize', () => {
+        globalThis.__paseoViewportTransitions.push({ width: innerWidth, height: innerHeight });
+      });
+      return globalThis.__paseoFocusContinuity;
+    }`,
  });
+  await page.evaluate((id) => {
+    const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
+    if (!webview) throw new Error(`Browser webview ${id} was unavailable`);
+    const events = [];
+    globalThis.__paseoBrowserReactivationEvents = events;
+    for (const name of [
+      "did-start-loading",
+      "did-navigate-in-page",
+      "load-commit",
+      "did-stop-loading",
+    ]) {
+      webview.addEventListener(name, (event) => {
+        events.push({
+          name,
+          url: event.url ?? null,
+          isMainFrame: event.isMainFrame ?? null,
+        });
+      });
+    }
+  }, browserId);

  await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).click();
  await page.waitForFunction(
@@ -458,6 +573,29 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId
    { id: browserId, webContentsId: firstGuest.webContentsId },
    { timeout: timeoutMs },
  );
+  await page.waitForTimeout(1_500);
+  const reactivationEvents = await page.evaluate(
+    () => globalThis.__paseoBrowserReactivationEvents ?? [],
+  );
+  const unexpectedReactivationCommits = reactivationEvents.filter(
+    (event) => event.name === "did-navigate-in-page" || event.name === "load-commit",
+  );
+  assert(
+    unexpectedReactivationCommits.length === 0,
+    `responsive browser reactivation committed the current document: ${JSON.stringify(unexpectedReactivationCommits)}`,
+  );
+  const viewportTransitionsResult = await callBrowserTool(client, "browser_evaluate", {
+    browserId,
+    function: "() => globalThis.__paseoViewportTransitions ?? []",
+  });
+  const viewportTransitions = JSON.parse(viewportTransitionsResult.resultJson);
+  const collapsedViewport = viewportTransitions.find(
+    (viewport) => viewport.width <= 1 || viewport.height <= 1,
+  );
+  assert(
+    collapsedViewport === undefined,
+    `responsive browser reactivation collapsed the guest viewport: ${JSON.stringify(viewportTransitions)}`,
+  );
  const continuityResult = await callBrowserTool(client, "browser_evaluate", {
    browserId,
    function: "() => globalThis.__paseoFocusContinuity ?? null",
@@ -531,7 +669,7 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId
    failures,
    "browser viewport survives workspace eviction and reattachment",
    await readViewport(client, browserId),
-    requestedViewport,
+    responsiveViewport,
  );

  await originalDeck.getByRole("button", { name: "Annotate element" }).click();
@@ -553,6 +691,8 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId
    originalWebContentsId: firstGuest.webContentsId,
    finalWebContentsId: parkedGuest.webContentsId,
    viewport: "passed",
+    guestFocus: "passed",
+    overlayPlane: "passed",
    inactiveCapture: "passed",
    list: "passed",
    snapshot: "passed",
@@ -463,6 +463,18 @@ ipcMain.handle("paseo:browser:set-workspace-active-browser", (event, rawInput: u
  }
 });

+ipcMain.handle("paseo:browser:focus", (event, browserId: unknown): boolean => {
+  if (typeof browserId !== "string" || browserId.trim().length === 0) {
+    return false;
+  }
+  const contents = getPaseoBrowserWebContentsForHostWindow(browserId, event.sender.id);
+  if (!contents) {
+    return false;
+  }
+  contents.focus();
+  return true;
+});
+
 ipcMain.handle("paseo:browser:open-devtools", (event, browserId: unknown) => {
  if (typeof browserId !== "string" || browserId.trim().length === 0) {
    const result = {
@@ -111,6 +111,7 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
      ipcRenderer.invoke("paseo:browser:unregister-workspace-browser", browserId),
    setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) =>
      ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input),
+    focus: (browserId: string) => ipcRenderer.invoke("paseo:browser:focus", browserId),
    openDevTools: (browserId: string) =>
      ipcRenderer.invoke("paseo:browser:open-devtools", browserId),
    clearProfile: (legacyBrowserIds: string[]) =>