From dc19b45f23208e3530e4b737b3815d5e4578d708 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 7 Aug 2026 22:34:44 +0200 Subject: [PATCH] fix(desktop): keep browser surfaces interactive --- docs/browser-capture-harness.md | 8 + .../desktop/browser/pane/index.electron.tsx | 22 ++- .../browser/resident-webviews.browser.test.ts | 33 +++- .../src/desktop/browser/resident-webviews.ts | 33 ++-- packages/app/src/desktop/host.ts | 1 + packages/app/src/lib/overlay-root.ts | 12 +- packages/desktop/e2e/browser-tabs.e2e.mjs | 146 +++++++++++++++++- packages/desktop/src/main.ts | 12 ++ packages/desktop/src/preload.ts | 1 + 9 files changed, 234 insertions(+), 34 deletions(-) diff --git a/docs/browser-capture-harness.md b/docs/browser-capture-harness.md index 0038774dc..734dda3c4 100644 --- a/docs/browser-capture-harness.md +++ b/docs/browser-capture-harness.md @@ -87,6 +87,14 @@ or parking changes that surface's geometry without reparenting the webview. The uses `left:0`, `top:0`, `width:1px`, `height:1px`, `overflow:hidden`, `opacity:1`, and `pointer-events:none`. The webview stays at its resolved logical viewport, defaulting to 1280x800 before first presentation, with `display:inline-flex` at `left:0`, `top:0`. +Presentation resolves responsive guests to the pane's exact pixel dimensions after the surface +has visible bounds. Do not apply percentage guest sizing against the parked surface: Electron +exposes the 1x1 parking geometry as a real guest resize before expanding it again. + +The permanent browser host and `overlay-root` are explicit sibling paint planes. The browser +plane stays below the overlay plane regardless of body insertion order; menus keep their relative +layering inside `overlay-root`. Activating a presented browser also focuses its registered guest +`WebContents` in main so macOS assigns keyboard first-responder ownership to the page. There is no renderer prep/restore handshake. Main disables guest background throttling once when the webview attaches, then screenshot capture uses the shared serialized queue, diff --git a/packages/app/src/desktop/browser/pane/index.electron.tsx b/packages/app/src/desktop/browser/pane/index.electron.tsx index 5daa3fbd5..a97e9ee92 100644 --- a/packages/app/src/desktop/browser/pane/index.electron.tsx +++ b/packages/app/src/desktop/browser/pane/index.electron.tsx @@ -58,7 +58,6 @@ import { useBrowserStore, } from "@/desktop/browser/store"; import { - applyBrowserWebviewViewport, applyInactiveBrowserWebviewViewport, isResidentBrowserWebviewReady, prepareBrowserWebview, @@ -787,8 +786,7 @@ export function BrowserPane({ } releaseResidentBrowserWebview(browserId, webview); if (isPresentedRef.current) { - applyBrowserWebviewViewport(webview, browserViewportRef.current); - presentBrowserWebview(browserId, webview, host, clip); + presentBrowserWebview(browserId, webview, host, clip, browserViewportRef.current); } else { applyInactiveBrowserWebviewViewport(browserId, webview, browserViewportRef.current); } @@ -799,7 +797,13 @@ export function BrowserPane({ if (!isPresentedRef.current) { return; } - presentBrowserWebview(browserIdRef.current, webview, host, clip); + presentBrowserWebview( + browserIdRef.current, + webview, + host, + clip, + browserViewportRef.current, + ); rememberResolvedBrowserWebviewSize(browserIdRef.current, webview); }); @@ -882,6 +886,13 @@ export function BrowserPane({ }; const handleWebviewFocus = () => { onFocusPane?.(); + webview.focus?.(); + const focusBrowser = getDesktopHost()?.browser?.focus; + if (typeof focusBrowser === "function") { + void focusBrowser(browserIdRef.current).catch((error) => { + console.error("[browser-webview] focus failed", error); + }); + } }; webview.addEventListener("did-start-loading", handleStartLoading); @@ -946,11 +957,10 @@ export function BrowserPane({ releaseResidentBrowserWebview(browserId, webview); return; } - applyBrowserWebviewViewport(webview, browserViewport); const host = webviewHostRef.current; const clip = webviewClipRef.current; if (host && clip) { - presentBrowserWebview(browserId, webview, host, clip); + presentBrowserWebview(browserId, webview, host, clip, browserViewport); } if (browserViewport.mode === "fixed") { rememberBrowserWebviewSize({ diff --git a/packages/app/src/desktop/browser/resident-webviews.browser.test.ts b/packages/app/src/desktop/browser/resident-webviews.browser.test.ts index 8d19983cd..868005817 100644 --- a/packages/app/src/desktop/browser/resident-webviews.browser.test.ts +++ b/packages/app/src/desktop/browser/resident-webviews.browser.test.ts @@ -1,6 +1,5 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { - applyBrowserWebviewViewport, applyInactiveBrowserWebviewViewport, type BrowserWebviewProfileHost, clearResidentBrowserWebviewsForTests, @@ -141,7 +140,9 @@ describe("resident browser webviews", () => { } const permanentParent = webview.parentElement; - presentBrowserWebview("browser-stable-parent", webview, anchor, clip); + presentBrowserWebview("browser-stable-parent", webview, anchor, clip, { + mode: "responsive", + }); expect(webview.parentElement).toBe(permanentParent); expect(permanentParent.style.left).toBe("40px"); expect(permanentParent.style.top).toBe("60px"); @@ -149,12 +150,17 @@ describe("resident browser webviews", () => { expect(permanentParent.style.height).toBe("480px"); expect(permanentParent.style.pointerEvents).toBe("auto"); expect(permanentParent.getAttribute("aria-hidden")).toBe("false"); + expect(webview.style.flex).toBe("0 0 auto"); + expect(webview.style.width).toBe("640px"); + expect(webview.style.height).toBe("480px"); releaseResidentBrowserWebview("browser-stable-parent", webview); expect(webview.parentElement).toBe(permanentParent); expectParkedSurface(permanentParent); - presentBrowserWebview("browser-stable-parent", webview, anchor, clip); + presentBrowserWebview("browser-stable-parent", webview, anchor, clip, { + mode: "responsive", + }); expect(webview.parentElement).toBe(permanentParent); }); @@ -167,7 +173,6 @@ describe("resident browser webviews", () => { if (!webview?.parentElement) { throw new Error("Expected resident browser surface"); } - applyBrowserWebviewViewport(webview, { mode: "fixed", width: 2560, height: 1440 }); const anchor = document.createElement("div"); const clip = document.createElement("div"); Object.defineProperty(anchor, "getBoundingClientRect", { @@ -177,7 +182,11 @@ describe("resident browser webviews", () => { value: () => ({ left: 100, top: 150, width: 800, height: 600 }), }); - presentBrowserWebview("browser-oversized", webview, anchor, clip); + presentBrowserWebview("browser-oversized", webview, anchor, clip, { + mode: "fixed", + width: 2560, + height: 1440, + }); expect(webview.parentElement.style.left).toBe("100px"); expect(webview.parentElement.style.top).toBe("150px"); @@ -359,8 +368,20 @@ describe("resident browser webviews", () => { it("applies an exact fixed viewport without a flex width override", () => { const webview = document.createElement("webview"); + const anchor = document.createElement("div"); + const clip = document.createElement("div"); + Object.defineProperty(anchor, "getBoundingClientRect", { + value: () => ({ left: 0, top: 0, width: 640, height: 480 }), + }); + Object.defineProperty(clip, "getBoundingClientRect", { + value: () => ({ left: 0, top: 0, width: 640, height: 480 }), + }); - applyBrowserWebviewViewport(webview, { mode: "fixed", width: 640, height: 480 }); + presentBrowserWebview("browser-fixed", webview, anchor, clip, { + mode: "fixed", + width: 640, + height: 480, + }); expect(webview.style.flex).toBe("0 0 auto"); expect(webview.style.width).toBe("640px"); diff --git a/packages/app/src/desktop/browser/resident-webviews.ts b/packages/app/src/desktop/browser/resident-webviews.ts index 65d4b3e02..0b5352584 100644 --- a/packages/app/src/desktop/browser/resident-webviews.ts +++ b/packages/app/src/desktop/browser/resident-webviews.ts @@ -4,6 +4,7 @@ import { type DesktopBrowserBridge, } from "@/desktop/host"; import type { BrowserViewport } from "@/desktop/browser/store"; +import { WEB_SURFACE_PLANE } from "@/lib/overlay-root"; const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews"; const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id"; @@ -108,7 +109,7 @@ function applyResidentHostParkingStyle(host: HTMLElement): void { host.style.opacity = "1"; host.style.pointerEvents = "none"; host.style.display = "block"; - host.style.zIndex = "0"; + host.style.zIndex = String(WEB_SURFACE_PLANE.browser); host.style.clipPath = ""; host.style.visibility = "visible"; host.style.transform = ""; @@ -220,24 +221,16 @@ export function rememberBrowserWebviewSize(input: { return dimensions; } -function applyBrowserWebviewDimensions(webview: HTMLElement, viewport: BrowserViewport): void { +function applyBrowserWebviewDimensions( + webview: HTMLElement, + dimensions: { width: number; height: number }, +): void { webview.style.display = "flex"; webview.style.border = "0"; webview.style.background = "transparent"; - if (viewport.mode === "responsive") { - webview.style.flex = "1"; - webview.style.width = "100%"; - webview.style.height = "100%"; - return; - } webview.style.flex = "0 0 auto"; - webview.style.width = `${viewport.width}px`; - webview.style.height = `${viewport.height}px`; -} - -export function applyBrowserWebviewViewport(webview: HTMLElement, viewport: BrowserViewport): void { - clearResidentWebviewParkingStyle(webview); - applyBrowserWebviewDimensions(webview, viewport); + webview.style.width = `${Math.max(1, Math.round(dimensions.width))}px`; + webview.style.height = `${Math.max(1, Math.round(dimensions.height))}px`; } export function applyInactiveBrowserWebviewViewport( @@ -256,6 +249,7 @@ export function presentBrowserWebview( webview: HTMLElement, anchor: HTMLElement, clip: HTMLElement, + viewport: BrowserViewport, ): void { const normalizedBrowserId = trimNonEmpty(browserId); if (!normalizedBrowserId) { @@ -297,6 +291,13 @@ export function presentBrowserWebview( surface.style.pointerEvents = hasVisibleArea ? "auto" : "none"; surface.style.display = "flex"; surface.style.visibility = "visible"; + clearResidentWebviewParkingStyle(webview); + applyBrowserWebviewDimensions( + webview, + viewport.mode === "responsive" + ? { width: anchorBounds.width, height: anchorBounds.height } + : viewport, + ); webview.style.position = "absolute"; webview.style.left = `${Math.round(anchorBounds.left - surfaceLeft)}px`; webview.style.top = `${Math.round(anchorBounds.top - surfaceTop)}px`; @@ -432,7 +433,7 @@ export function resizeResidentBrowserWebview(input: { const ownerDocument = readDocument(); const webview = ownerDocument ? findBrowserWebview(normalizedBrowserId, ownerDocument) : null; if (webview) { - applyBrowserWebviewDimensions(webview, { mode: "fixed", ...dimensions }); + applyBrowserWebviewDimensions(webview, dimensions); } return dimensions; diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index 2c8c17680..9a0031857 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -151,6 +151,7 @@ export interface DesktopBrowserBridge { workspaceId: string; browserId: string | null; }) => Promise; + focus?: (browserId: string) => Promise; openDevTools?: (browserId: string) => Promise; clearProfile?: (legacyBrowserIds: string[]) => Promise; executeAutomationCommand?: ( diff --git a/packages/app/src/lib/overlay-root.ts b/packages/app/src/lib/overlay-root.ts index 9b19b8825..71ea30af1 100644 --- a/packages/app/src/lib/overlay-root.ts +++ b/packages/app/src/lib/overlay-root.ts @@ -28,14 +28,20 @@ export function getOverlayRoot(): HTMLElement { if (!el) { el = document.createElement("div"); el.id = "overlay-root"; - el.style.position = "fixed"; - el.style.inset = "0"; - el.style.pointerEvents = "none"; document.body.appendChild(el); } + el.style.position = "fixed"; + el.style.inset = "0"; + el.style.pointerEvents = "none"; + el.style.zIndex = String(WEB_SURFACE_PLANE.overlay); return el; } +export const WEB_SURFACE_PLANE = { + browser: 0, + overlay: 1, +} as const; + export const OVERLAY_Z = { floating: 10, modal: 20, diff --git a/packages/desktop/e2e/browser-tabs.e2e.mjs b/packages/desktop/e2e/browser-tabs.e2e.mjs index 6283f1fb0..ba175ad25 100644 --- a/packages/desktop/e2e/browser-tabs.e2e.mjs +++ b/packages/desktop/e2e/browser-tabs.e2e.mjs @@ -199,6 +199,8 @@ async function startTargetPage() { Desktop browser target + + `); }); @@ -323,6 +325,55 @@ async function readViewport(client, browserId) { return JSON.parse(evaluated.resultJson); } +async function clickGuestElement(page, client, browserId, selector) { + const evaluated = await callBrowserTool(client, "browser_evaluate", { + browserId, + function: `() => { + const element = document.querySelector(${JSON.stringify(selector)}); + if (!(element instanceof HTMLElement)) return null; + const rect = element.getBoundingClientRect(); + return { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; + }`, + }); + const elementRect = JSON.parse(evaluated.resultJson); + assert(elementRect, `Guest element ${selector} was unavailable`); + const webviewRect = await page.evaluate((id) => { + const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`); + if (!(webview instanceof HTMLElement)) return null; + const rect = webview.getBoundingClientRect(); + return { x: rect.x, y: rect.y }; + }, browserId); + assert(webviewRect, `Browser webview ${browserId} was unavailable`); + await page.mouse.click( + webviewRect.x + elementRect.x + elementRect.width / 2, + webviewRect.y + elementRect.y + elementRect.height / 2, + ); +} + +async function selectDeviceSize(page, label) { + await page.locator('[aria-label="Device size"]').click(); + const item = page.getByText(label, { exact: true }); + await item.waitFor({ state: "visible", timeout: timeoutMs }); + const rect = await item.boundingBox(); + assert(rect, `Device size menu item ${label} had no bounds`); + const clip = { + x: Math.max(0, rect.x), + y: Math.max(0, rect.y), + width: rect.width, + height: rect.height, + }; + const openPixels = await page.screenshot({ clip }); + await page.keyboard.press("Escape"); + await item.waitFor({ state: "hidden", timeout: timeoutMs }); + const closedPixels = await page.screenshot({ clip }); + + await page.locator('[aria-label="Device size"]').click(); + await item.waitFor({ state: "visible", timeout: timeoutMs }); + await page.mouse.click(rect.x + rect.width / 2, rect.y + rect.height / 2); + await page.keyboard.press("Escape"); + return !openPixels.equals(closedPixels); +} + function recordViewportMismatch(failures, label, actual, expected) { if (actual.width === expected.width && actual.height === expected.height) { return; @@ -341,6 +392,16 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId await originalWorkspaceRow.waitFor({ state: "visible", timeout: timeoutMs }); await originalWorkspaceRow.click(); + await page.evaluate(() => { + if (document.getElementById("overlay-root")) return; + const overlayRoot = document.createElement("div"); + overlayRoot.id = "overlay-root"; + overlayRoot.style.position = "fixed"; + overlayRoot.style.inset = "0"; + overlayRoot.style.pointerEvents = "none"; + document.body.appendChild(overlayRoot); + }); + const created = await callBrowserTool(client, "browser_new_tab", { url: targetUrl }); const browserId = created.browserId; assert(typeof browserId === "string", "browser_new_tab returned no browserId"); @@ -364,6 +425,30 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId { width: firstGuest.width, height: firstGuest.height }, ); + await clickGuestElement(page, client, browserId, "#typing-target"); + const activeGuestElement = await callBrowserTool(client, "browser_evaluate", { + browserId, + function: "() => document.activeElement?.id ?? null", + }); + assert( + JSON.parse(activeGuestElement.resultJson) === "typing-target", + "Physical browser click did not focus the guest input", + ); + const focusedGuest = await page.evaluate( + (id) => window.paseoDesktop?.browser?.focus?.(id), + browserId, + ); + assert(focusedGuest === true, "Electron did not focus the registered browser guest"); + + const deviceSizeMenuPainted = await selectDeviceSize(page, "iPhone SE · 375×667"); + assert(deviceSizeMenuPainted, "Device size menu did not paint above the browser surface"); + recordViewportMismatch( + failures, + "device size menu paints and receives input above the browser surface", + await readViewport(client, browserId), + { width: 375, height: 667 }, + ); + await callBrowserTool(client, "browser_wait", { browserId, text: "Bridge target", @@ -426,6 +511,9 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId } await callBrowserTool(client, "browser_resize", { browserId, ...requestedViewport }); + await selectDeviceSize(page, "Responsive"); + const responsiveViewport = await readViewport(client, browserId); + await originalDeck.getByTestId(`workspace-tab-agent_${callerAgentId}`).click(); await page.waitForTimeout(500); try { @@ -437,13 +525,40 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId failures, "inactive browser preserves the shared viewport", await readViewport(client, browserId), - requestedViewport, + responsiveViewport, ); const focusContinuitySentinel = "preserve-browser-document-across-focus"; await callBrowserTool(client, "browser_evaluate", { browserId, - function: `() => { globalThis.__paseoFocusContinuity = ${JSON.stringify(focusContinuitySentinel)}; return globalThis.__paseoFocusContinuity; }`, + function: `() => { + globalThis.__paseoFocusContinuity = ${JSON.stringify(focusContinuitySentinel)}; + globalThis.__paseoViewportTransitions = [{ width: innerWidth, height: innerHeight }]; + addEventListener('resize', () => { + globalThis.__paseoViewportTransitions.push({ width: innerWidth, height: innerHeight }); + }); + return globalThis.__paseoFocusContinuity; + }`, }); + await page.evaluate((id) => { + const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`); + if (!webview) throw new Error(`Browser webview ${id} was unavailable`); + const events = []; + globalThis.__paseoBrowserReactivationEvents = events; + for (const name of [ + "did-start-loading", + "did-navigate-in-page", + "load-commit", + "did-stop-loading", + ]) { + webview.addEventListener(name, (event) => { + events.push({ + name, + url: event.url ?? null, + isMainFrame: event.isMainFrame ?? null, + }); + }); + } + }, browserId); await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).click(); await page.waitForFunction( @@ -458,6 +573,29 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId { id: browserId, webContentsId: firstGuest.webContentsId }, { timeout: timeoutMs }, ); + await page.waitForTimeout(1_500); + const reactivationEvents = await page.evaluate( + () => globalThis.__paseoBrowserReactivationEvents ?? [], + ); + const unexpectedReactivationCommits = reactivationEvents.filter( + (event) => event.name === "did-navigate-in-page" || event.name === "load-commit", + ); + assert( + unexpectedReactivationCommits.length === 0, + `responsive browser reactivation committed the current document: ${JSON.stringify(unexpectedReactivationCommits)}`, + ); + const viewportTransitionsResult = await callBrowserTool(client, "browser_evaluate", { + browserId, + function: "() => globalThis.__paseoViewportTransitions ?? []", + }); + const viewportTransitions = JSON.parse(viewportTransitionsResult.resultJson); + const collapsedViewport = viewportTransitions.find( + (viewport) => viewport.width <= 1 || viewport.height <= 1, + ); + assert( + collapsedViewport === undefined, + `responsive browser reactivation collapsed the guest viewport: ${JSON.stringify(viewportTransitions)}`, + ); const continuityResult = await callBrowserTool(client, "browser_evaluate", { browserId, function: "() => globalThis.__paseoFocusContinuity ?? null", @@ -531,7 +669,7 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId failures, "browser viewport survives workspace eviction and reattachment", await readViewport(client, browserId), - requestedViewport, + responsiveViewport, ); await originalDeck.getByRole("button", { name: "Annotate element" }).click(); @@ -553,6 +691,8 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId originalWebContentsId: firstGuest.webContentsId, finalWebContentsId: parkedGuest.webContentsId, viewport: "passed", + guestFocus: "passed", + overlayPlane: "passed", inactiveCapture: "passed", list: "passed", snapshot: "passed", diff --git a/packages/desktop/src/main.ts b/packages/desktop/src/main.ts index 07621fb7f..36b66783a 100644 --- a/packages/desktop/src/main.ts +++ b/packages/desktop/src/main.ts @@ -463,6 +463,18 @@ ipcMain.handle("paseo:browser:set-workspace-active-browser", (event, rawInput: u } }); +ipcMain.handle("paseo:browser:focus", (event, browserId: unknown): boolean => { + if (typeof browserId !== "string" || browserId.trim().length === 0) { + return false; + } + const contents = getPaseoBrowserWebContentsForHostWindow(browserId, event.sender.id); + if (!contents) { + return false; + } + contents.focus(); + return true; +}); + ipcMain.handle("paseo:browser:open-devtools", (event, browserId: unknown) => { if (typeof browserId !== "string" || browserId.trim().length === 0) { const result = { diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index 1267d2233..fc3bb19ec 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -111,6 +111,7 @@ contextBridge.exposeInMainWorld("paseoDesktop", { ipcRenderer.invoke("paseo:browser:unregister-workspace-browser", browserId), setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) => ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input), + focus: (browserId: string) => ipcRenderer.invoke("paseo:browser:focus", browserId), openDevTools: (browserId: string) => ipcRenderer.invoke("paseo:browser:open-devtools", browserId), clearProfile: (legacyBrowserIds: string[]) =>