Keep desktop browser tabs stable across focus and automation (#2907)

* fix(desktop): preserve browser tabs across focus changes

Electron can replace a guest WebContents when a webview is reparented, which reloads the page and can race automation against a destroyed target. Keep each browser guest on one permanent paintable surface and make user and agent resizing share one canonical viewport.

* fix(desktop): recover browser surfaces after lifecycle changes

* fix(desktop): preserve resident browser lifecycle state

* fix(desktop): seed inactive browser viewport
This commit is contained in:
Mohamed Boudra
2026-08-06 00:05:21 +02:00
committed by GitHub
parent 7f7e8923eb
commit eb95a6eea6
12 changed files with 929 additions and 84 deletions
+6 -5
View File
@@ -5,7 +5,7 @@ It validates the compositor behavior that unit tests cannot see:
- the resident automation `<webview>` starts in the production parking state; - the resident automation `<webview>` starts in the production parking state;
- the parked guest remains paintable and has a copyable viewport frame; - the parked guest remains paintable and has a copyable viewport frame;
- the resident webview guest is sized to 1280x800 logical pixels; - a never-presented resident webview guest defaults to 1280x800 logical pixels;
- multiple resident webviews are parked as an overlapping stack without per-capture - multiple resident webviews are parked as an overlapping stack without per-capture
stacking changes; stacking changes;
- a newly attached resident webview whose first useful frame is delayed can be captured - a newly attached resident webview whose first useful frame is delayed can be captured
@@ -82,10 +82,11 @@ is usually saved as 2560x1600.
Electron captures copy from the guest web contents' compositor surface. A resident Electron captures copy from the guest web contents' compositor surface. A resident
webview parked with `display:none`, offscreen coordinates, or `opacity:0` can lose its webview parked with `display:none`, offscreen coordinates, or `opacity:0` can lose its
copyable surface. The production parking state keeps the host fixed at `left:0`, `top:0`, copyable surface. Each production webview keeps one permanent body-level surface. Presenting
`width:1px`, `height:1px`, `overflow:hidden`, `opacity:1`, and `pointer-events:none`. or parking changes that surface's geometry without reparenting the webview. The parking state
The webviews inside stay full-size at 1280x800, `display:inline-flex`, and absolutely uses `left:0`, `top:0`, `width:1px`, `height:1px`, `overflow:hidden`, `opacity:1`, and
overlap at `left:0`, `top:0`. `pointer-events:none`. The webview stays at its resolved logical viewport, defaulting to
1280x800 before first presentation, with `display:inline-flex` at `left:0`, `top:0`.
There is no renderer prep/restore handshake. Main disables guest background throttling There is no renderer prep/restore handshake. Main disables guest background throttling
once when the webview attaches, then screenshot capture uses the shared serialized queue, once when the webview attaches, then screenshot capture uses the shared serialized queue,
@@ -411,6 +411,11 @@ describe("mountBrowserAutomationHandler", () => {
height: 768, height: 768,
}, },
}); });
expect(useBrowserStore.getState().browsersById[result.browserId]?.viewport).toEqual({
mode: "fixed",
width: 1024,
height: 768,
});
expect(browser.browser.executedRequests).toHaveLength(1); expect(browser.browser.executedRequests).toHaveLength(1);
}); });
@@ -5,7 +5,12 @@ import {
removeResidentBrowserWebview, removeResidentBrowserWebview,
resizeResidentBrowserWebview, resizeResidentBrowserWebview,
} from "@/desktop/browser/resident-webviews"; } from "@/desktop/browser/resident-webviews";
import { createWorkspaceBrowser, getBrowserRecord, useBrowserStore } from "@/desktop/browser/store"; import {
createFixedBrowserViewport,
createWorkspaceBrowser,
getBrowserRecord,
useBrowserStore,
} from "@/desktop/browser/store";
import { collectAllTabs, useWorkspaceLayoutStore } from "@/stores/workspace-layout-store"; import { collectAllTabs, useWorkspaceLayoutStore } from "@/stores/workspace-layout-store";
import { buildWorkspaceTabPersistenceKey } from "@/workspace-tabs/model"; import { buildWorkspaceTabPersistenceKey } from "@/workspace-tabs/model";
@@ -208,6 +213,9 @@ function resizeBrowserTabForRequest(params: {
message: `No browser tab found for ID: ${browserId}`, message: `No browser tab found for ID: ${browserId}`,
}); });
} }
useBrowserStore
.getState()
.setBrowserViewport(browserId, createFixedBrowserViewport(dimensions.width, dimensions.height));
return { return {
requestId: request.requestId, requestId: request.requestId,
@@ -28,6 +28,7 @@ import { StyleSheet, useUnistyles, withUnistyles } from "react-native-unistyles"
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import * as Clipboard from "expo-clipboard"; import * as Clipboard from "expo-clipboard";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { useRetainedPanelActive } from "@/components/retained-panel";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { useToast } from "@/contexts/toast-context"; import { useToast } from "@/contexts/toast-context";
import { import {
@@ -49,10 +50,22 @@ import {
isElectronRuntime, isElectronRuntime,
type DesktopBrowserShortcutEvent, type DesktopBrowserShortcutEvent,
} from "@/desktop/host"; } from "@/desktop/host";
import { useBrowserStore, normalizeWorkspaceBrowserUrl } from "@/desktop/browser/store";
import { import {
type BrowserViewport,
createFixedBrowserViewport,
normalizeWorkspaceBrowserUrl,
RESPONSIVE_BROWSER_VIEWPORT,
useBrowserStore,
} from "@/desktop/browser/store";
import {
applyBrowserWebviewViewport,
applyInactiveBrowserWebviewViewport,
isResidentBrowserWebviewReady,
prepareBrowserWebview, prepareBrowserWebview,
presentBrowserWebview,
rememberBrowserWebviewSize,
releaseResidentBrowserWebview, releaseResidentBrowserWebview,
removeResidentBrowserWebview,
takeResidentBrowserWebview, takeResidentBrowserWebview,
} from "../resident-webviews"; } from "../resident-webviews";
@@ -551,7 +564,7 @@ function DeviceSizeMenu({
onSelect, onSelect,
triggerStyle, triggerStyle,
}: { }: {
selectedId: DeviceSizeId; selectedId: DeviceSizeId | null;
onSelect: (id: DeviceSizeId) => void; onSelect: (id: DeviceSizeId) => void;
triggerStyle: (state: { hovered?: boolean; pressed?: boolean }) => StyleProp<ViewStyle>; triggerStyle: (state: { hovered?: boolean; pressed?: boolean }) => StyleProp<ViewStyle>;
}) { }) {
@@ -590,6 +603,22 @@ function DeviceSizeMenu({
); );
} }
function deviceSizeIdForViewport(viewport: BrowserViewport): DeviceSizeId | null {
if (viewport.mode === "responsive") {
return "responsive";
}
return (
DEVICE_SIZE_PRESETS.find(
(preset) => preset.width === viewport.width && preset.height === viewport.height,
)?.id ?? null
);
}
function rememberResolvedBrowserWebviewSize(browserId: string, webview: HTMLElement): void {
const bounds = webview.getBoundingClientRect();
rememberBrowserWebviewSize({ browserId, width: bounds.width, height: bounds.height });
}
// eslint-disable-next-line complexity // eslint-disable-next-line complexity
export function BrowserPane({ export function BrowserPane({
browserId, browserId,
@@ -610,8 +639,16 @@ export function BrowserPane({
const { t } = useTranslation(); const { t } = useTranslation();
const browser = useBrowserStore((state) => state.browsersById[browserId] ?? null); const browser = useBrowserStore((state) => state.browsersById[browserId] ?? null);
const updateBrowser = useBrowserStore((state) => state.updateBrowser); const updateBrowser = useBrowserStore((state) => state.updateBrowser);
const setBrowserViewport = useBrowserStore((state) => state.setBrowserViewport);
const browserViewport = browser?.viewport ?? RESPONSIVE_BROWSER_VIEWPORT;
const browserViewportRef = useRef(browserViewport);
browserViewportRef.current = browserViewport;
const isPresented = useRetainedPanelActive();
const isPresentedRef = useRef(isPresented);
isPresentedRef.current = isPresented;
const webviewRef = useRef<ElectronWebview | null>(null); const webviewRef = useRef<ElectronWebview | null>(null);
const webviewHostRef = useRef<HTMLDivElement | null>(null); const webviewHostRef = useRef<HTMLDivElement | null>(null);
const webviewClipRef = useRef<HTMLElement | null>(null);
const urlInputRef = useRef<WebTextInput | null>(null); const urlInputRef = useRef<WebTextInput | null>(null);
const initialUrlRef = useRef(browser?.url ?? "https://example.com"); const initialUrlRef = useRef(browser?.url ?? "https://example.com");
const browserIdRef = useRef(browserId); const browserIdRef = useRef(browserId);
@@ -629,7 +666,6 @@ export function BrowserPane({
const toast = useToast(); const toast = useToast();
const toastRef = useRef(toast); const toastRef = useRef(toast);
toastRef.current = toast; toastRef.current = toast;
const [deviceSizeId, setDeviceSizeId] = useState<DeviceSizeId>("responsive");
const [pendingSelection, setPendingSelection] = useState<BrowserElementSelection | null>(null); const [pendingSelection, setPendingSelection] = useState<BrowserElementSelection | null>(null);
// Screenshot is captured at selection time (overlay already torn down, no // Screenshot is captured at selection time (overlay already torn down, no
// scroll drift) and reused when the annotation card is submitted. // scroll drift) and reused when the annotation card is submitted.
@@ -727,7 +763,8 @@ export function BrowserPane({
} }
const host = webviewHostRef.current; const host = webviewHostRef.current;
if (!host) { const clip = webviewClipRef.current;
if (!host || !clip) {
return; return;
} }
@@ -740,6 +777,7 @@ export function BrowserPane({
const residentWebview = takeResidentBrowserWebview(browserId) as ElectronWebview | null; const residentWebview = takeResidentBrowserWebview(browserId) as ElectronWebview | null;
const webview = residentWebview ?? (document.createElement("webview") as ElectronWebview); const webview = residentWebview ?? (document.createElement("webview") as ElectronWebview);
webviewRef.current = webview; webviewRef.current = webview;
domReadyRef.current = isResidentBrowserWebviewReady(webview);
if (!residentWebview) { if (!residentWebview) {
prepareBrowserWebview(webview, { prepareBrowserWebview(webview, {
browserId, browserId,
@@ -747,14 +785,26 @@ export function BrowserPane({
initialUrl: initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current, initialUrl: initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current,
}); });
} }
webview.style.display = "flex"; releaseResidentBrowserWebview(browserId, webview);
webview.style.flex = "1"; if (isPresentedRef.current) {
webview.style.width = "100%"; applyBrowserWebviewViewport(webview, browserViewportRef.current);
webview.style.height = "100%"; presentBrowserWebview(browserId, webview, host, clip);
webview.style.border = "0"; } else {
webview.style.background = "transparent"; applyInactiveBrowserWebviewViewport(browserId, webview, browserViewportRef.current);
}
const sizeObserver =
typeof ResizeObserver === "undefined"
? null
: new ResizeObserver(() => {
if (!isPresentedRef.current) {
return;
}
presentBrowserWebview(browserIdRef.current, webview, host, clip);
rememberResolvedBrowserWebviewSize(browserIdRef.current, webview);
});
const handleStartLoading = () => { const handleStartLoading = () => {
domReadyRef.current = false;
updateBrowser(browserId, { isLoading: true, lastError: null }); updateBrowser(browserId, { isLoading: true, lastError: null });
syncNavigationState({ syncUrl: false }); syncNavigationState({ syncUrl: false });
}; };
@@ -846,7 +896,11 @@ export function BrowserPane({
webview.addEventListener("focus", handleWebviewFocus); webview.addEventListener("focus", handleWebviewFocus);
webview.addEventListener("mousedown", handleWebviewFocus); webview.addEventListener("mousedown", handleWebviewFocus);
host.appendChild(webview); if (isPresentedRef.current) {
rememberResolvedBrowserWebviewSize(browserId, webview);
}
sizeObserver?.observe(host);
sizeObserver?.observe(clip);
if (initialUnsafeNavigationMessage) { if (initialUnsafeNavigationMessage) {
updateBrowserRef.current(browserIdRef.current, { updateBrowserRef.current(browserIdRef.current, {
isLoading: false, isLoading: false,
@@ -855,6 +909,7 @@ export function BrowserPane({
} }
return () => { return () => {
sizeObserver?.disconnect();
webview.removeEventListener("did-start-loading", handleStartLoading); webview.removeEventListener("did-start-loading", handleStartLoading);
webview.removeEventListener("did-stop-loading", handleStopLoading); webview.removeEventListener("did-stop-loading", handleStopLoading);
webview.removeEventListener("will-navigate", handleWillNavigate); webview.removeEventListener("will-navigate", handleWillNavigate);
@@ -866,15 +921,13 @@ export function BrowserPane({
webview.removeEventListener("dom-ready", handleDomReady); webview.removeEventListener("dom-ready", handleDomReady);
webview.removeEventListener("focus", handleWebviewFocus); webview.removeEventListener("focus", handleWebviewFocus);
webview.removeEventListener("mousedown", handleWebviewFocus); webview.removeEventListener("mousedown", handleWebviewFocus);
if (host.contains(webview)) { const browserStillExists = Boolean(
const browserStillExists = Boolean( useBrowserStore.getState().browsersById[browserIdRef.current],
useBrowserStore.getState().browsersById[browserIdRef.current], );
); if (browserStillExists) {
if (browserStillExists) { releaseResidentBrowserWebview(browserIdRef.current, webview);
releaseResidentBrowserWebview(browserIdRef.current, webview); } else {
} else { removeResidentBrowserWebview(browserIdRef.current);
host.removeChild(webview);
}
} }
if (webviewRef.current === webview) { if (webviewRef.current === webview) {
webviewRef.current = null; webviewRef.current = null;
@@ -884,6 +937,32 @@ export function BrowserPane({
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
}, [browserId, onFocusPane]); }, [browserId, onFocusPane]);
useEffect(() => {
const webview = webviewRef.current;
if (!webview) {
return;
}
if (!isPresented) {
releaseResidentBrowserWebview(browserId, webview);
return;
}
applyBrowserWebviewViewport(webview, browserViewport);
const host = webviewHostRef.current;
const clip = webviewClipRef.current;
if (host && clip) {
presentBrowserWebview(browserId, webview, host, clip);
}
if (browserViewport.mode === "fixed") {
rememberBrowserWebviewSize({
browserId,
width: browserViewport.width,
height: browserViewport.height,
});
} else {
rememberResolvedBrowserWebviewSize(browserId, webview);
}
}, [browserId, browserViewport, isPresented]);
const navigate = useCallback( const navigate = useCallback(
(nextUrl: string) => { (nextUrl: string) => {
const normalizedUrl = normalizeWorkspaceBrowserUrl(nextUrl); const normalizedUrl = normalizeWorkspaceBrowserUrl(nextUrl);
@@ -1525,12 +1604,26 @@ export function BrowserPane({
[selectorMode], [selectorMode],
); );
const devicePreset = useMemo( const selectedDeviceSizeId = useMemo(
() => () => deviceSizeIdForViewport(browserViewport),
DEVICE_SIZE_PRESETS.find((preset) => preset.id === deviceSizeId) ?? DEVICE_SIZE_PRESETS[0], [browserViewport],
[deviceSizeId], );
const isResponsiveDevice = browserViewport.mode === "responsive";
const handleSelectDeviceSize = useCallback(
(deviceSizeId: DeviceSizeId) => {
const preset =
DEVICE_SIZE_PRESETS.find((candidate) => candidate.id === deviceSizeId) ??
DEVICE_SIZE_PRESETS[0];
setBrowserViewport(
browserId,
preset.width === null || preset.height === null
? RESPONSIVE_BROWSER_VIEWPORT
: createFixedBrowserViewport(preset.width, preset.height),
);
},
[browserId, setBrowserViewport],
); );
const isResponsiveDevice = devicePreset.width === null;
const webviewHostStyle = useMemo<CSSProperties>( const webviewHostStyle = useMemo<CSSProperties>(
() => () =>
@@ -1546,15 +1639,13 @@ export function BrowserPane({
: { : {
// Fixed-size device frame, centered within webviewWrap (see styles). // Fixed-size device frame, centered within webviewWrap (see styles).
display: "flex", display: "flex",
width: devicePreset.width ?? undefined, width: browserViewport.width,
maxWidth: "100%", height: browserViewport.height,
height: devicePreset.height ?? undefined,
maxHeight: "100%",
minHeight: 0, minHeight: 0,
background: theme.colors.surface0, background: theme.colors.surface0,
boxShadow: "0 2px 16px rgba(0,0,0,0.25)", boxShadow: "0 2px 16px rgba(0,0,0,0.25)",
}, },
[devicePreset.height, devicePreset.width, isResponsiveDevice, theme.colors.surface0], [browserViewport, isResponsiveDevice, theme.colors.surface0],
); );
const webviewWrapStyle = useMemo( const webviewWrapStyle = useMemo(
@@ -1566,6 +1657,10 @@ export function BrowserPane({
webviewHostRef.current = node; webviewHostRef.current = node;
}, []); }, []);
const setWebviewClipNode = useCallback((node: unknown) => {
webviewClipRef.current = node instanceof HTMLElement ? node : null;
}, []);
if (!isElectronRuntime()) { if (!isElectronRuntime()) {
return ( return (
<View style={styles.unavailableState}> <View style={styles.unavailableState}>
@@ -1624,8 +1719,8 @@ export function BrowserPane({
</View> </View>
<View style={styles.chromeRight}> <View style={styles.chromeRight}>
<DeviceSizeMenu <DeviceSizeMenu
selectedId={deviceSizeId} selectedId={selectedDeviceSizeId}
onSelect={setDeviceSizeId} onSelect={handleSelectDeviceSize}
triggerStyle={baseIconButtonStyle} triggerStyle={baseIconButtonStyle}
/> />
<ToolbarButton <ToolbarButton
@@ -1678,7 +1773,11 @@ export function BrowserPane({
</Text> </Text>
</View> </View>
) : null} ) : null}
<View style={webviewWrapStyle}> <View
ref={setWebviewClipNode}
style={webviewWrapStyle}
testID={`browser-webview-clip-${browserId}`}
>
{createElement("div", { {createElement("div", {
ref: setWebviewHostNode, ref: setWebviewHostNode,
style: webviewHostStyle, style: webviewHostStyle,
@@ -1884,6 +1983,7 @@ const styles = StyleSheet.create((theme) => ({
}, },
annotationOverlay: { annotationOverlay: {
position: "absolute", position: "absolute",
zIndex: 1,
left: 0, left: 0,
right: 0, right: 0,
bottom: 0, bottom: 0,
@@ -1,12 +1,18 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { import {
applyBrowserWebviewViewport,
applyInactiveBrowserWebviewViewport,
type BrowserWebviewProfileHost, type BrowserWebviewProfileHost,
clearResidentBrowserWebviewsForTests, clearResidentBrowserWebviewsForTests,
ensureResidentBrowserWebview, ensureResidentBrowserWebview,
getResidentBrowserWebview, getResidentBrowserWebview,
isResidentBrowserWebviewReady,
prepareBrowserWebview, prepareBrowserWebview,
presentBrowserWebview,
rememberBrowserWebviewSize,
releaseResidentBrowserWebview, releaseResidentBrowserWebview,
removeResidentBrowserWebview, removeResidentBrowserWebview,
resizeResidentBrowserWebview,
takeResidentBrowserWebview, takeResidentBrowserWebview,
} from "./resident-webviews"; } from "./resident-webviews";
import { import {
@@ -54,14 +60,28 @@ function expectPermanentHostParking(host: HTMLElement): void {
expect(host.style.position).toBe("fixed"); expect(host.style.position).toBe("fixed");
expect(host.style.left).toBe("0px"); expect(host.style.left).toBe("0px");
expect(host.style.top).toBe("0px"); expect(host.style.top).toBe("0px");
expect(host.style.width).toBe("1px"); expect(host.style.width).toBe("100vw");
expect(host.style.height).toBe("1px"); expect(host.style.height).toBe("100vh");
expect(host.style.overflow).toBe("hidden"); expect(host.style.overflow).toBe("visible");
expect(host.style.opacity).toBe("1"); expect(host.style.opacity).toBe("1");
expect(host.style.pointerEvents).toBe("none"); expect(host.style.pointerEvents).toBe("none");
expect(host.style.display).toBe("block"); expect(host.style.display).toBe("block");
expect(host.style.visibility).toBe("visible"); expect(host.style.zIndex).toBe("0");
expect(host.style.transform).toBe(""); }
function expectParkedSurface(surface: HTMLElement): void {
expect(surface.getAttribute("aria-hidden")).toBe("true");
expect(surface.style.position).toBe("fixed");
expect(surface.style.left).toBe("0px");
expect(surface.style.top).toBe("0px");
expect(surface.style.width).toBe("1px");
expect(surface.style.height).toBe("1px");
expect(surface.style.overflow).toBe("hidden");
expect(surface.style.opacity).toBe("1");
expect(surface.style.pointerEvents).toBe("none");
expect(surface.style.display).toBe("block");
expect(surface.style.visibility).toBe("visible");
expect(surface.style.transform).toBe("");
} }
function expectResidentWebviewParking(webview: HTMLElement): void { function expectResidentWebviewParking(webview: HTMLElement): void {
@@ -94,12 +114,110 @@ describe("resident browser webviews", () => {
const host = residentHost(); const host = residentHost();
expect(visibleHost.children).toHaveLength(0); expect(visibleHost.children).toHaveLength(0);
expect(Array.from(host.children)).toEqual([webview]); const surface = host.firstElementChild as HTMLElement;
expect(Array.from(surface.children)).toEqual([webview]);
expect(webview.isConnected).toBe(true); expect(webview.isConnected).toBe(true);
expectPermanentHostParking(host); expectPermanentHostParking(host);
expectParkedSurface(surface);
expectResidentWebviewParking(webview); expectResidentWebviewParking(webview);
}); });
it("presents and parks a browser without changing its webview parent", () => {
const webview = ensureTestBrowser({
browserId: "browser-stable-parent",
workspaceId: "workspace-stable-parent",
url: "https://example.com",
});
const anchor = document.createElement("div");
const clip = document.createElement("div");
Object.defineProperty(anchor, "getBoundingClientRect", {
value: () => ({ left: 40, top: 60, width: 640, height: 480 }),
});
Object.defineProperty(clip, "getBoundingClientRect", {
value: () => ({ left: 40, top: 60, width: 640, height: 480 }),
});
if (!webview?.parentElement) {
throw new Error("Expected resident browser surface");
}
const permanentParent = webview.parentElement;
presentBrowserWebview("browser-stable-parent", webview, anchor, clip);
expect(webview.parentElement).toBe(permanentParent);
expect(permanentParent.style.left).toBe("40px");
expect(permanentParent.style.top).toBe("60px");
expect(permanentParent.style.width).toBe("640px");
expect(permanentParent.style.height).toBe("480px");
expect(permanentParent.style.pointerEvents).toBe("auto");
expect(permanentParent.getAttribute("aria-hidden")).toBe("false");
releaseResidentBrowserWebview("browser-stable-parent", webview);
expect(webview.parentElement).toBe(permanentParent);
expectParkedSurface(permanentParent);
presentBrowserWebview("browser-stable-parent", webview, anchor, clip);
expect(webview.parentElement).toBe(permanentParent);
});
it("clips an oversized fixed viewport to its pane without resizing the webview", () => {
const webview = ensureTestBrowser({
browserId: "browser-oversized",
workspaceId: "workspace-oversized",
url: "https://example.com",
});
if (!webview?.parentElement) {
throw new Error("Expected resident browser surface");
}
applyBrowserWebviewViewport(webview, { mode: "fixed", width: 2560, height: 1440 });
const anchor = document.createElement("div");
const clip = document.createElement("div");
Object.defineProperty(anchor, "getBoundingClientRect", {
value: () => ({ left: -800, top: -300, width: 2560, height: 1440 }),
});
Object.defineProperty(clip, "getBoundingClientRect", {
value: () => ({ left: 100, top: 150, width: 800, height: 600 }),
});
presentBrowserWebview("browser-oversized", webview, anchor, clip);
expect(webview.parentElement.style.left).toBe("100px");
expect(webview.parentElement.style.top).toBe("150px");
expect(webview.parentElement.style.width).toBe("800px");
expect(webview.parentElement.style.height).toBe("600px");
expect(webview.style.left).toBe("-900px");
expect(webview.style.top).toBe("-450px");
expect(webview.style.width).toBe("2560px");
expect(webview.style.height).toBe("1440px");
resizeResidentBrowserWebview({ browserId: "browser-oversized", width: 2560, height: 1440 });
expect(webview.style.left).toBe("-900px");
expect(webview.style.top).toBe("-450px");
expect(webview.parentElement.style.width).toBe("800px");
expect(webview.parentElement.style.height).toBe("600px");
});
it("retains document readiness with the resident webview", () => {
const webview = ensureTestBrowser({
browserId: "browser-ready",
workspaceId: "workspace-ready",
url: "https://example.com",
});
if (!webview) {
throw new Error("Expected resident browser webview");
}
expect(isResidentBrowserWebviewReady(webview)).toBe(false);
webview.dispatchEvent(new Event("dom-ready"));
releaseResidentBrowserWebview("browser-ready", webview);
const reusedWebview = takeResidentBrowserWebview("browser-ready");
expect(reusedWebview).toBe(webview);
expect(isResidentBrowserWebviewReady(reusedWebview as HTMLElement)).toBe(true);
webview.dispatchEvent(new Event("did-start-loading"));
expect(isResidentBrowserWebviewReady(webview)).toBe(false);
});
it("creates a resident webview for an agent-created unfocused tab", () => { it("creates a resident webview for an agent-created unfocused tab", () => {
const webview = ensureTestBrowser({ const webview = ensureTestBrowser({
browserId: "browser-agent", browserId: "browser-agent",
@@ -197,8 +315,10 @@ describe("resident browser webviews", () => {
}); });
expect(webview).toBe(staleWebview); expect(webview).toBe(staleWebview);
expect(Array.from(staleHost.children)).toEqual([staleWebview]); const surface = staleHost.firstElementChild as HTMLElement;
expect(Array.from(surface.children)).toEqual([staleWebview]);
expectPermanentHostParking(staleHost); expectPermanentHostParking(staleHost);
expectParkedSurface(surface);
expectResidentWebviewParking(staleWebview); expectResidentWebviewParking(staleWebview);
}); });
@@ -215,27 +335,73 @@ describe("resident browser webviews", () => {
}); });
const host = residentHost(); const host = residentHost();
expect(firstWebview?.parentElement).toBe(host); expect(firstWebview?.parentElement?.parentElement).toBe(host);
expect(secondWebview?.parentElement).toBe(host); expect(secondWebview?.parentElement?.parentElement).toBe(host);
expect(firstWebview?.parentElement).not.toBe(secondWebview?.parentElement);
expectResidentWebviewParking(firstWebview as HTMLElement); expectResidentWebviewParking(firstWebview as HTMLElement);
expectResidentWebviewParking(secondWebview as HTMLElement); expectResidentWebviewParking(secondWebview as HTMLElement);
}); });
it("moves a resident webview into a visible pane without recreating the node", () => { it("returns a resident webview without detaching it from its permanent surface", () => {
const webview = ensureTestBrowser({ const webview = ensureTestBrowser({
browserId: "browser-visible", browserId: "browser-visible",
workspaceId: "workspace-visible", workspaceId: "workspace-visible",
url: "https://example.com", url: "https://example.com",
}); });
const permanentParent = webview?.parentElement;
const visibleWebview = takeResidentBrowserWebview("browser-visible"); const visibleWebview = takeResidentBrowserWebview("browser-visible");
expect(visibleWebview).toBe(webview); expect(visibleWebview).toBe(webview);
expect(webview?.style.position).toBe(""); expect(webview?.parentElement).toBe(permanentParent);
expect(webview?.style.left).toBe(""); expect(takeResidentBrowserWebview("browser-visible")).toBe(webview);
expect(webview?.style.top).toBe(""); });
expect(webview?.style.zIndex).toBe("");
expect(takeResidentBrowserWebview("browser-visible")).toBeNull(); it("applies an exact fixed viewport without a flex width override", () => {
const webview = document.createElement("webview");
applyBrowserWebviewViewport(webview, { mode: "fixed", width: 640, height: 480 });
expect(webview.style.flex).toBe("0 0 auto");
expect(webview.style.width).toBe("640px");
expect(webview.style.height).toBe("480px");
});
it("parks a browser at its last resolved viewport dimensions", () => {
const visibleHost = document.createElement("div");
const webview = document.createElement("webview");
visibleHost.appendChild(webview);
document.body.appendChild(visibleHost);
rememberBrowserWebviewSize({ browserId: "browser-sized", width: 640, height: 480 });
releaseResidentBrowserWebview("browser-sized", webview);
expect(webview.style.width).toBe("640px");
expect(webview.style.height).toBe("480px");
});
it("keeps an inactive in-place browser at its last resolved viewport dimensions", () => {
const webview = document.createElement("webview");
rememberBrowserWebviewSize({ browserId: "browser-inactive", width: 640, height: 480 });
applyInactiveBrowserWebviewViewport("browser-inactive", webview, { mode: "responsive" });
expect(webview.style.flex).toBe("0 0 auto");
expect(webview.style.width).toBe("640px");
expect(webview.style.height).toBe("480px");
});
it("parks a cold inactive browser at its canonical fixed viewport", () => {
const webview = document.createElement("webview");
applyInactiveBrowserWebviewViewport("browser-restored-fixed", webview, {
mode: "fixed",
width: 800,
height: 600,
});
expect(webview.style.width).toBe("800px");
expect(webview.style.height).toBe("600px");
}); });
it("returns an existing visible pane webview instead of creating a resident duplicate", () => { it("returns an existing visible pane webview instead of creating a resident duplicate", () => {
@@ -3,14 +3,18 @@ import {
type DesktopAttachedBrowserRegistration, type DesktopAttachedBrowserRegistration,
type DesktopBrowserBridge, type DesktopBrowserBridge,
} from "@/desktop/host"; } from "@/desktop/host";
import type { BrowserViewport } from "@/desktop/browser/store";
const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews"; const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id"; const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
const BROWSER_SURFACE_ATTRIBUTE = "data-paseo-browser-surface";
const RESIDENT_VIEWPORT_WIDTH = 1280; const RESIDENT_VIEWPORT_WIDTH = 1280;
const RESIDENT_VIEWPORT_HEIGHT = 800; const RESIDENT_VIEWPORT_HEIGHT = 800;
const residentWebviewsByBrowserId = new Map<string, HTMLElement>(); const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
const residentSurfacesByBrowserId = new Map<string, HTMLElement>();
const residentWebviewSizesByBrowserId = new Map<string, { width: number; height: number }>(); const residentWebviewSizesByBrowserId = new Map<string, { width: number; height: number }>();
const readyResidentWebviews = new WeakSet<HTMLElement>();
interface BrowserWebviewElement extends HTMLElement { interface BrowserWebviewElement extends HTMLElement {
src: string; src: string;
@@ -70,6 +74,15 @@ function registerBrowserWhenAttached(
}); });
} }
function registerBrowserReadiness(webview: HTMLElement): void {
webview.addEventListener("did-start-loading", () => {
readyResidentWebviews.delete(webview);
});
webview.addEventListener("dom-ready", () => {
readyResidentWebviews.add(webview);
});
}
function trimNonEmpty(value: string | null | undefined): string | null { function trimNonEmpty(value: string | null | undefined): string | null {
if (typeof value !== "string") { if (typeof value !== "string") {
return null; return null;
@@ -83,24 +96,52 @@ function readDocument(): Document | null {
} }
function applyResidentHostParkingStyle(host: HTMLElement): void { function applyResidentHostParkingStyle(host: HTMLElement): void {
// Parked browser webviews must remain paintable at all times; screenshot // The host is permanent. Individual browser surfaces switch between their
// correctness depends on the proven states in docs/browser-capture-harness.md. // pane bounds and the proven paintable 1x1 parking geometry.
host.setAttribute("aria-hidden", "true"); host.removeAttribute("aria-hidden");
host.style.position = "fixed"; host.style.position = "fixed";
host.style.left = "0"; host.style.left = "0";
host.style.top = "0"; host.style.top = "0";
host.style.width = "1px"; host.style.width = "100vw";
host.style.height = "1px"; host.style.height = "100vh";
host.style.overflow = "hidden"; host.style.overflow = "visible";
host.style.opacity = "1"; host.style.opacity = "1";
host.style.pointerEvents = "none"; host.style.pointerEvents = "none";
host.style.display = "block"; host.style.display = "block";
host.style.zIndex = ""; host.style.zIndex = "0";
host.style.clipPath = ""; host.style.clipPath = "";
host.style.visibility = "visible"; host.style.visibility = "visible";
host.style.transform = ""; host.style.transform = "";
} }
function applyParkedBrowserSurfaceStyle(surface: HTMLElement): void {
surface.setAttribute("aria-hidden", "true");
surface.style.position = "fixed";
surface.style.left = "0";
surface.style.top = "0";
surface.style.width = "1px";
surface.style.height = "1px";
surface.style.overflow = "hidden";
surface.style.opacity = "1";
surface.style.pointerEvents = "none";
surface.style.display = "block";
surface.style.visibility = "visible";
surface.style.transform = "";
}
function getBrowserSurface(browserId: string, ownerDocument: Document): HTMLElement {
const existing = residentSurfacesByBrowserId.get(browserId);
if (existing?.isConnected) {
return existing;
}
const surface = ownerDocument.createElement("div");
surface.setAttribute(BROWSER_SURFACE_ATTRIBUTE, browserId);
applyParkedBrowserSurfaceStyle(surface);
getResidentBrowserHost(ownerDocument).appendChild(surface);
residentSurfacesByBrowserId.set(browserId, surface);
return surface;
}
function getResidentBrowserHost(ownerDocument: Document): HTMLElement { function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID); const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
if (existing) { if (existing) {
@@ -162,6 +203,105 @@ function clearResidentWebviewParkingStyle(webview: HTMLElement): void {
webview.style.zIndex = ""; webview.style.zIndex = "";
} }
export function rememberBrowserWebviewSize(input: {
browserId: string;
width: number;
height: number;
}): { width: number; height: number } | null {
const browserId = trimNonEmpty(input.browserId);
if (!browserId || input.width <= 0 || input.height <= 0) {
return null;
}
const dimensions = {
width: Math.max(1, Math.round(input.width)),
height: Math.max(1, Math.round(input.height)),
};
residentWebviewSizesByBrowserId.set(browserId, dimensions);
return dimensions;
}
function applyBrowserWebviewDimensions(webview: HTMLElement, viewport: BrowserViewport): void {
webview.style.display = "flex";
webview.style.border = "0";
webview.style.background = "transparent";
if (viewport.mode === "responsive") {
webview.style.flex = "1";
webview.style.width = "100%";
webview.style.height = "100%";
return;
}
webview.style.flex = "0 0 auto";
webview.style.width = `${viewport.width}px`;
webview.style.height = `${viewport.height}px`;
}
export function applyBrowserWebviewViewport(webview: HTMLElement, viewport: BrowserViewport): void {
clearResidentWebviewParkingStyle(webview);
applyBrowserWebviewDimensions(webview, viewport);
}
export function applyInactiveBrowserWebviewViewport(
browserId: string,
webview: HTMLElement,
viewport: BrowserViewport,
): void {
if (viewport.mode === "fixed") {
rememberBrowserWebviewSize({ browserId, width: viewport.width, height: viewport.height });
}
applyResidentWebviewStyle(webview, trimNonEmpty(browserId));
}
export function presentBrowserWebview(
browserId: string,
webview: HTMLElement,
anchor: HTMLElement,
clip: HTMLElement,
): void {
const normalizedBrowserId = trimNonEmpty(browserId);
if (!normalizedBrowserId) {
return;
}
const ownerDocument = readDocument();
if (!ownerDocument) {
return;
}
const surface = getBrowserSurface(normalizedBrowserId, ownerDocument);
if (webview.parentElement !== surface) {
surface.appendChild(webview);
}
const anchorBounds = anchor.getBoundingClientRect();
const clipBounds = clip.getBoundingClientRect();
const left = Math.max(anchorBounds.left, clipBounds.left);
const top = Math.max(anchorBounds.top, clipBounds.top);
const right = Math.min(
anchorBounds.left + anchorBounds.width,
clipBounds.left + clipBounds.width,
);
const bottom = Math.min(
anchorBounds.top + anchorBounds.height,
clipBounds.top + clipBounds.height,
);
const surfaceLeft = Math.ceil(left);
const surfaceTop = Math.ceil(top);
const surfaceRight = Math.floor(right);
const surfaceBottom = Math.floor(bottom);
const hasVisibleArea = surfaceRight > surfaceLeft && surfaceBottom > surfaceTop;
surface.setAttribute("aria-hidden", "false");
surface.style.position = "fixed";
surface.style.left = `${surfaceLeft}px`;
surface.style.top = `${surfaceTop}px`;
surface.style.width = `${Math.max(0, surfaceRight - surfaceLeft)}px`;
surface.style.height = `${Math.max(0, surfaceBottom - surfaceTop)}px`;
surface.style.overflow = "hidden";
surface.style.opacity = "1";
surface.style.pointerEvents = hasVisibleArea ? "auto" : "none";
surface.style.display = "flex";
surface.style.visibility = "visible";
webview.style.position = "absolute";
webview.style.left = `${Math.round(anchorBounds.left - surfaceLeft)}px`;
webview.style.top = `${Math.round(anchorBounds.top - surfaceTop)}px`;
}
export function prepareBrowserWebview( export function prepareBrowserWebview(
webview: HTMLElement, webview: HTMLElement,
input: { input: {
@@ -181,6 +321,11 @@ export function prepareBrowserWebview(
(webview as BrowserWebviewElement).src = input.initialUrl; (webview as BrowserWebviewElement).src = input.initialUrl;
} }
registerBrowserWhenAttached(webview as BrowserWebviewElement, input, browser); registerBrowserWhenAttached(webview as BrowserWebviewElement, input, browser);
registerBrowserReadiness(webview);
}
export function isResidentBrowserWebviewReady(webview: HTMLElement): boolean {
return readyResidentWebviews.has(webview);
} }
export function ensureResidentBrowserWebview(input: { export function ensureResidentBrowserWebview(input: {
@@ -247,8 +392,6 @@ export function takeResidentBrowserWebview(browserId: string): HTMLElement | nul
return null; return null;
} }
residentWebviewsByBrowserId.delete(normalizedBrowserId);
clearResidentWebviewParkingStyle(webview);
return webview; return webview;
} }
@@ -265,7 +408,11 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl
residentWebviewsByBrowserId.set(normalizedBrowserId, webview); residentWebviewsByBrowserId.set(normalizedBrowserId, webview);
applyResidentWebviewStyle(webview, normalizedBrowserId); applyResidentWebviewStyle(webview, normalizedBrowserId);
getResidentBrowserHost(ownerDocument).appendChild(webview); const surface = getBrowserSurface(normalizedBrowserId, ownerDocument);
applyParkedBrowserSurfaceStyle(surface);
if (webview.parentElement !== surface) {
surface.appendChild(webview);
}
} }
export function resizeResidentBrowserWebview(input: { export function resizeResidentBrowserWebview(input: {
@@ -277,18 +424,18 @@ export function resizeResidentBrowserWebview(input: {
if (!normalizedBrowserId) { if (!normalizedBrowserId) {
return null; return null;
} }
const width = Math.max(1, Math.round(input.width)); const dimensions = rememberBrowserWebviewSize(input);
const height = Math.max(1, Math.round(input.height)); if (!dimensions) {
residentWebviewSizesByBrowserId.set(normalizedBrowserId, { width, height }); return null;
}
const ownerDocument = readDocument(); const ownerDocument = readDocument();
const webview = ownerDocument ? findBrowserWebview(normalizedBrowserId, ownerDocument) : null; const webview = ownerDocument ? findBrowserWebview(normalizedBrowserId, ownerDocument) : null;
if (webview) { if (webview) {
webview.style.width = `${width}px`; applyBrowserWebviewDimensions(webview, { mode: "fixed", ...dimensions });
webview.style.height = `${height}px`;
} }
return { width, height }; return dimensions;
} }
export function removeResidentBrowserWebview(browserId: string): void { export function removeResidentBrowserWebview(browserId: string): void {
@@ -298,9 +445,12 @@ export function removeResidentBrowserWebview(browserId: string): void {
} }
const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null; const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
const surface = residentSurfacesByBrowserId.get(normalizedBrowserId) ?? null;
residentWebviewsByBrowserId.delete(normalizedBrowserId); residentWebviewsByBrowserId.delete(normalizedBrowserId);
residentSurfacesByBrowserId.delete(normalizedBrowserId);
residentWebviewSizesByBrowserId.delete(normalizedBrowserId); residentWebviewSizesByBrowserId.delete(normalizedBrowserId);
resident?.remove(); resident?.remove();
surface?.remove();
} }
export function clearResidentBrowserWebviewsForTests(): void { export function clearResidentBrowserWebviewsForTests(): void {
@@ -308,6 +458,7 @@ export function clearResidentBrowserWebviewsForTests(): void {
webview.remove(); webview.remove();
} }
residentWebviewsByBrowserId.clear(); residentWebviewsByBrowserId.clear();
residentSurfacesByBrowserId.clear();
residentWebviewSizesByBrowserId.clear(); residentWebviewSizesByBrowserId.clear();
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove(); readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
} }
@@ -7,18 +7,26 @@ import {
type BrowserIndexState, type BrowserIndexState,
type BrowserRecord, type BrowserRecord,
type BrowserRecordPatch, type BrowserRecordPatch,
type BrowserViewport,
createBrowserRecord, createBrowserRecord,
normalizeBrowserIndexState,
normalizeBrowserUrl, normalizeBrowserUrl,
removeBrowserFromIndex, removeBrowserFromIndex,
sanitizeBrowsersForPersist, sanitizeBrowsersForPersist,
trimNonEmpty, trimNonEmpty,
} from "./state"; } from "./state";
export type { BrowserRecord } from "./state"; export {
createFixedBrowserViewport,
RESPONSIVE_BROWSER_VIEWPORT,
type BrowserRecord,
type BrowserViewport,
} from "./state";
interface BrowserStoreState extends BrowserIndexState { interface BrowserStoreState extends BrowserIndexState {
createBrowser: (input?: { initialUrl?: string }) => string; createBrowser: (input?: { initialUrl?: string }) => string;
updateBrowser: (browserId: string, patch: BrowserRecordPatch) => void; updateBrowser: (browserId: string, patch: BrowserRecordPatch) => void;
setBrowserViewport: (browserId: string, viewport: BrowserViewport) => void;
removeBrowser: (browserId: string) => void; removeBrowser: (browserId: string) => void;
} }
@@ -57,6 +65,9 @@ export const useBrowserStore = create<BrowserStoreState>()(
updateBrowser: (browserId, patch) => { updateBrowser: (browserId, patch) => {
set((state) => applyBrowserPatch(state, browserId, patch)); set((state) => applyBrowserPatch(state, browserId, patch));
}, },
setBrowserViewport: (browserId, viewport) => {
set((state) => applyBrowserPatch(state, browserId, { viewport }));
},
removeBrowser: (browserId) => { removeBrowser: (browserId) => {
set((state) => removeBrowserFromIndex(state, browserId)); set((state) => removeBrowserFromIndex(state, browserId));
}, },
@@ -65,6 +76,10 @@ export const useBrowserStore = create<BrowserStoreState>()(
name: "workspace-browser-store", name: "workspace-browser-store",
storage: createJSONStorage(() => AsyncStorage), storage: createJSONStorage(() => AsyncStorage),
partialize: (state) => sanitizeBrowsersForPersist(state), partialize: (state) => sanitizeBrowsersForPersist(state),
merge: (persistedState, currentState) => ({
...currentState,
...normalizeBrowserIndexState(persistedState),
}),
}, },
), ),
); );
@@ -2,7 +2,9 @@ import { describe, expect, it } from "vitest";
import { import {
applyBrowserPatch, applyBrowserPatch,
type BrowserIndexState, type BrowserIndexState,
createFixedBrowserViewport,
createBrowserRecord, createBrowserRecord,
normalizeBrowserIndexState,
normalizeBrowserUrl, normalizeBrowserUrl,
removeBrowserFromIndex, removeBrowserFromIndex,
sanitizeBrowsersForPersist, sanitizeBrowsersForPersist,
@@ -57,12 +59,25 @@ describe("createBrowserRecord", () => {
canGoForward: false, canGoForward: false,
faviconUrl: null, faviconUrl: null,
lastError: null, lastError: null,
viewport: { mode: "responsive" },
createdAt: 1000, createdAt: 1000,
}); });
}); });
}); });
describe("applyBrowserPatch", () => { describe("applyBrowserPatch", () => {
it("updates the browser's canonical viewport", () => {
const initial = withRecords([
createBrowserRecord({ browserId: "b1", initialUrl: "https://a.test", now: 0 }),
]);
const next = applyBrowserPatch(initial, "b1", {
viewport: createFixedBrowserViewport(639.6, 479.6),
});
expect(next.browsersById.b1?.viewport).toEqual({ mode: "fixed", width: 640, height: 480 });
});
it("normalizes URL updates", () => { it("normalizes URL updates", () => {
const initial = withRecords([ const initial = withRecords([
createBrowserRecord({ browserId: "b1", initialUrl: "https://a.test", now: 0 }), createBrowserRecord({ browserId: "b1", initialUrl: "https://a.test", now: 0 }),
@@ -145,4 +160,32 @@ describe("sanitizeBrowsersForPersist", () => {
expect(persisted.browsersById.b1?.isLoading).toBe(false); expect(persisted.browsersById.b1?.isLoading).toBe(false);
expect(persisted.browsersById.b1?.lastError).toBe(null); expect(persisted.browsersById.b1?.lastError).toBe(null);
}); });
it("preserves viewport state", () => {
const base = createBrowserRecord({ browserId: "b1", initialUrl: "https://a.test", now: 0 });
const state = applyBrowserPatch(withRecords([base]), "b1", {
viewport: createFixedBrowserViewport(640, 480),
});
expect(sanitizeBrowsersForPersist(state).browsersById.b1?.viewport).toEqual({
mode: "fixed",
width: 640,
height: 480,
});
});
});
describe("normalizeBrowserIndexState", () => {
it("defaults legacy persisted records to Responsive", () => {
const legacy = createBrowserRecord({
browserId: "b1",
initialUrl: "https://a.test",
now: 0,
}) as Partial<ReturnType<typeof createBrowserRecord>>;
delete legacy.viewport;
expect(
normalizeBrowserIndexState({ browsersById: { b1: legacy } }).browsersById.b1?.viewport,
).toEqual({ mode: "responsive" });
});
}); });
@@ -1,3 +1,9 @@
export type BrowserViewport =
| { mode: "responsive" }
| { mode: "fixed"; width: number; height: number };
export const RESPONSIVE_BROWSER_VIEWPORT: BrowserViewport = { mode: "responsive" };
export interface BrowserRecord { export interface BrowserRecord {
browserId: string; browserId: string;
url: string; url: string;
@@ -7,6 +13,7 @@ export interface BrowserRecord {
canGoForward: boolean; canGoForward: boolean;
faviconUrl: string | null; faviconUrl: string | null;
lastError: string | null; lastError: string | null;
viewport: BrowserViewport;
createdAt: number; createdAt: number;
} }
@@ -16,6 +23,62 @@ export interface BrowserIndexState {
browsersById: Record<string, BrowserRecord>; browsersById: Record<string, BrowserRecord>;
} }
export function createFixedBrowserViewport(width: number, height: number): BrowserViewport {
return {
mode: "fixed",
width: Math.max(1, Math.round(width)),
height: Math.max(1, Math.round(height)),
};
}
export function normalizeBrowserViewport(value: unknown): BrowserViewport {
if (
value &&
typeof value === "object" &&
(value as { mode?: unknown }).mode === "fixed" &&
typeof (value as { width?: unknown }).width === "number" &&
typeof (value as { height?: unknown }).height === "number"
) {
return createFixedBrowserViewport(
(value as { width: number }).width,
(value as { height: number }).height,
);
}
return RESPONSIVE_BROWSER_VIEWPORT;
}
function browserViewportsEqual(left: BrowserViewport, right: BrowserViewport): boolean {
return (
left.mode === right.mode &&
(left.mode === "responsive" ||
(right.mode === "fixed" && left.width === right.width && left.height === right.height))
);
}
export function normalizeBrowserIndexState(value: unknown): BrowserIndexState {
if (!value || typeof value !== "object") {
return { browsersById: {} };
}
const browsersById = (value as { browsersById?: unknown }).browsersById;
if (!browsersById || typeof browsersById !== "object") {
return { browsersById: {} };
}
return {
browsersById: Object.fromEntries(
Object.entries(browsersById).map(([browserId, record]) => {
const browser = record as BrowserRecord & { viewport?: unknown };
return [
browserId,
{
...browser,
viewport: normalizeBrowserViewport(browser.viewport),
} satisfies BrowserRecord,
];
}),
),
};
}
export function trimNonEmpty(value: string | null | undefined): string | null { export function trimNonEmpty(value: string | null | undefined): string | null {
if (typeof value !== "string") { if (typeof value !== "string") {
return null; return null;
@@ -55,6 +118,7 @@ export function createBrowserRecord(input: {
canGoForward: false, canGoForward: false,
faviconUrl: null, faviconUrl: null,
lastError: null, lastError: null,
viewport: RESPONSIVE_BROWSER_VIEWPORT,
createdAt: input.now, createdAt: input.now,
}; };
} }
@@ -73,9 +137,13 @@ export function applyBrowserPatch<S extends BrowserIndexState>(
return state; return state;
} }
const nextViewport = patch.viewport
? normalizeBrowserViewport(patch.viewport)
: existing.viewport;
const nextRecord: BrowserRecord = { const nextRecord: BrowserRecord = {
...existing, ...existing,
...patch, ...patch,
viewport: nextViewport,
url: normalizeBrowserUrl(patch.url ?? existing.url), url: normalizeBrowserUrl(patch.url ?? existing.url),
}; };
@@ -86,7 +154,8 @@ export function applyBrowserPatch<S extends BrowserIndexState>(
nextRecord.canGoBack === existing.canGoBack && nextRecord.canGoBack === existing.canGoBack &&
nextRecord.canGoForward === existing.canGoForward && nextRecord.canGoForward === existing.canGoForward &&
nextRecord.faviconUrl === existing.faviconUrl && nextRecord.faviconUrl === existing.faviconUrl &&
nextRecord.lastError === existing.lastError nextRecord.lastError === existing.lastError &&
browserViewportsEqual(nextRecord.viewport, existing.viewport)
) { ) {
return state; return state;
} }
+215 -8
View File
@@ -272,11 +272,68 @@ async function readGuest(page, browserId) {
return { return {
webContentsId: webview.getWebContentsId(), webContentsId: webview.getWebContentsId(),
parentId: webview.parentElement?.id ?? null, parentId: webview.parentElement?.id ?? null,
width: Math.round(webview.getBoundingClientRect().width),
height: Math.round(webview.getBoundingClientRect().height),
}; };
}, browserId); }, browserId);
} }
async function runRegression({ page, client, serverId, targetUrl }) { async function readPresentation(page, browserId) {
return await page.evaluate((id) => {
const surface = document.querySelector(`[data-paseo-browser-surface="${id}"]`);
const clip = document.querySelector(`[data-testid="browser-webview-clip-${id}"]`);
if (!(surface instanceof HTMLElement) || !(clip instanceof HTMLElement)) return null;
const surfaceRect = surface.getBoundingClientRect();
const clipRect = clip.getBoundingClientRect();
const webview = surface.querySelector(`[data-paseo-browser-id="${id}"]`);
if (!(webview instanceof HTMLElement)) return null;
const webviewRect = webview.getBoundingClientRect();
const outsidePoint = {
x: Math.max(0, Math.round(clipRect.left - 1)),
y: Math.round(clipRect.top + clipRect.height / 2),
};
const outsideTarget = document.elementFromPoint(outsidePoint.x, outsidePoint.y);
return {
surface: {
left: Math.round(surfaceRect.left),
top: Math.round(surfaceRect.top),
right: Math.round(surfaceRect.right),
bottom: Math.round(surfaceRect.bottom),
},
clip: {
left: Math.round(clipRect.left),
top: Math.round(clipRect.top),
right: Math.round(clipRect.right),
bottom: Math.round(clipRect.bottom),
},
webview: {
left: Math.round(webviewRect.left),
top: Math.round(webviewRect.top),
},
capturesOutsideInput: surface.contains(outsideTarget),
};
}, browserId);
}
async function readViewport(client, browserId) {
const evaluated = await callBrowserTool(client, "browser_evaluate", {
browserId,
function: "() => ({ width: window.innerWidth, height: window.innerHeight })",
});
return JSON.parse(evaluated.resultJson);
}
function recordViewportMismatch(failures, label, actual, expected) {
if (actual.width === expected.width && actual.height === expected.height) {
return;
}
failures.push(
`${label}: expected ${expected.width}x${expected.height}, received ${actual.width}x${actual.height}`,
);
}
async function runRegression({ page, client, serverId, targetUrl, callerAgentId }) {
const failures = [];
const originalWorkspaceId = workspaceIds[0]; const originalWorkspaceId = workspaceIds[0];
const originalWorkspaceRow = page.getByTestId( const originalWorkspaceRow = page.getByTestId(
`sidebar-workspace-row-${serverId}:${originalWorkspaceId}`, `sidebar-workspace-row-${serverId}:${originalWorkspaceId}`,
@@ -300,6 +357,117 @@ async function runRegression({ page, client, serverId, targetUrl }) {
); );
const firstGuest = await readGuest(page, browserId); const firstGuest = await readGuest(page, browserId);
assert(firstGuest, "Original browser guest was not attached to its workspace pane"); assert(firstGuest, "Original browser guest was not attached to its workspace pane");
recordViewportMismatch(
failures,
"Responsive viewport follows the visible browser pane",
await readViewport(client, browserId),
{ width: firstGuest.width, height: firstGuest.height },
);
await callBrowserTool(client, "browser_wait", {
browserId,
text: "Bridge target",
timeoutMs: 5_000,
});
const requestedViewport = { width: 640, height: 480 };
await callBrowserTool(client, "browser_resize", { browserId, ...requestedViewport });
recordViewportMismatch(
failures,
"browser_resize updates the visible shared viewport",
await readViewport(client, browserId),
requestedViewport,
);
const oversizedViewport = { width: 2560, height: 1440 };
await callBrowserTool(client, "browser_resize", { browserId, ...oversizedViewport });
recordViewportMismatch(
failures,
"oversized preset preserves the requested guest viewport",
await readViewport(client, browserId),
oversizedViewport,
);
await page.waitForFunction(
({ id, width, height }) => {
const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
return (
webview instanceof HTMLElement &&
Math.round(webview.getBoundingClientRect().width) === width &&
Math.round(webview.getBoundingClientRect().height) === height
);
},
{ id: browserId, ...oversizedViewport },
{ timeout: timeoutMs },
);
const presentation = await readPresentation(page, browserId);
assert(presentation, "Oversized browser presentation geometry was unavailable");
if (
presentation.surface.left < presentation.clip.left ||
presentation.surface.top < presentation.clip.top ||
presentation.surface.right > presentation.clip.right ||
presentation.surface.bottom > presentation.clip.bottom
) {
failures.push(
`oversized browser surface stays inside its pane: ${JSON.stringify(presentation)}`,
);
}
if (presentation.capturesOutsideInput) {
failures.push("oversized browser surface captures input outside its pane");
}
await callBrowserTool(client, "browser_resize", { browserId, ...oversizedViewport });
const repeatedResizePresentation = await readPresentation(page, browserId);
assert(repeatedResizePresentation, "Repeated resize presentation geometry was unavailable");
if (
repeatedResizePresentation.webview.left !== presentation.webview.left ||
repeatedResizePresentation.webview.top !== presentation.webview.top
) {
failures.push(
`repeating an oversized resize preserves the guest offset: before ${JSON.stringify(presentation.webview)}, after ${JSON.stringify(repeatedResizePresentation.webview)}`,
);
}
await callBrowserTool(client, "browser_resize", { browserId, ...requestedViewport });
await originalDeck.getByTestId(`workspace-tab-agent_${callerAgentId}`).click();
await page.waitForTimeout(500);
try {
await callBrowserTool(client, "browser_screenshot", { browserId });
} catch (error) {
failures.push(`inactive browser remains captureable: ${String(error)}`);
}
recordViewportMismatch(
failures,
"inactive browser preserves the shared viewport",
await readViewport(client, browserId),
requestedViewport,
);
const focusContinuitySentinel = "preserve-browser-document-across-focus";
await callBrowserTool(client, "browser_evaluate", {
browserId,
function: `() => { globalThis.__paseoFocusContinuity = ${JSON.stringify(focusContinuitySentinel)}; return globalThis.__paseoFocusContinuity; }`,
});
await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).click();
await page.waitForFunction(
({ id, webContentsId }) => {
const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
return (
webview?.parentElement?.getAttribute("data-paseo-browser-surface") === id &&
webview.parentElement.style.pointerEvents === "auto" &&
webview.getWebContentsId() === webContentsId
);
},
{ id: browserId, webContentsId: firstGuest.webContentsId },
{ timeout: timeoutMs },
);
const continuityResult = await callBrowserTool(client, "browser_evaluate", {
browserId,
function: "() => globalThis.__paseoFocusContinuity ?? null",
});
const continuityValue = JSON.parse(continuityResult.resultJson);
if (continuityValue !== focusContinuitySentinel) {
failures.push(
`focusing the browser tab preserves the current document: expected ${JSON.stringify(focusContinuitySentinel)}, received ${JSON.stringify(continuityValue)}`,
);
}
for (const workspaceId of workspaceIds.slice(1)) { for (const workspaceId of workspaceIds.slice(1)) {
await page.getByTestId(`sidebar-workspace-row-${serverId}:${workspaceId}`).click(); await page.getByTestId(`sidebar-workspace-row-${serverId}:${workspaceId}`).click();
@@ -312,21 +480,22 @@ async function runRegression({ page, client, serverId, targetUrl }) {
({ id, previousWebContentsId }) => { ({ id, previousWebContentsId }) => {
const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`); const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
return ( return (
webview?.parentElement?.id === "paseo-browser-resident-webviews" && webview?.parentElement?.getAttribute("data-paseo-browser-surface") === id &&
webview.parentElement.style.width === "1px" &&
typeof webview.getWebContentsId === "function" && typeof webview.getWebContentsId === "function" &&
webview.getWebContentsId() !== previousWebContentsId webview.getWebContentsId() === previousWebContentsId
); );
}, },
{ id: browserId, previousWebContentsId: firstGuest.webContentsId }, { id: browserId, previousWebContentsId: firstGuest.webContentsId },
{ timeout: timeoutMs }, { timeout: timeoutMs },
); );
const replacementGuest = await readGuest(page, browserId); const parkedGuest = await readGuest(page, browserId);
assert(replacementGuest, "Replacement browser guest was not parked after workspace eviction"); assert(parkedGuest, "Browser guest was not parked after workspace eviction");
const listed = await callBrowserTool(client, "browser_list_tabs"); const listed = await callBrowserTool(client, "browser_list_tabs");
assert( assert(
listed.tabs.some((tab) => tab.browserId === browserId), listed.tabs.some((tab) => tab.browserId === browserId),
"browser_list_tabs lost the original tab after guest replacement", "browser_list_tabs lost the original tab after workspace eviction",
); );
const snapshot = await callBrowserTool(client, "browser_snapshot", { browserId }); const snapshot = await callBrowserTool(client, "browser_snapshot", { browserId });
@@ -344,10 +513,47 @@ async function runRegression({ page, client, serverId, targetUrl }) {
timeoutMs: 5_000, timeoutMs: 5_000,
}); });
await originalWorkspaceRow.click();
await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).click();
await page.waitForFunction(
({ id, webContentsId }) => {
const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
return (
webview?.parentElement?.getAttribute("data-paseo-browser-surface") === id &&
webview.parentElement.style.pointerEvents === "auto" &&
webview.getWebContentsId() === webContentsId
);
},
{ id: browserId, webContentsId: firstGuest.webContentsId },
{ timeout: timeoutMs },
);
recordViewportMismatch(
failures,
"browser viewport survives workspace eviction and reattachment",
await readViewport(client, browserId),
requestedViewport,
);
await originalDeck.getByRole("button", { name: "Annotate element" }).click();
await page.waitForTimeout(250);
const selectorResult = await callBrowserTool(client, "browser_evaluate", {
browserId,
function: "() => Boolean(globalThis.__paseoSelector)",
});
if (JSON.parse(selectorResult.resultJson) !== true) {
failures.push("reused loaded browser remains ready for element annotation after remount");
}
if (failures.length > 0) {
throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`);
}
return { return {
browserId, browserId,
originalWebContentsId: firstGuest.webContentsId, originalWebContentsId: firstGuest.webContentsId,
replacementWebContentsId: replacementGuest.webContentsId, finalWebContentsId: parkedGuest.webContentsId,
viewport: "passed",
inactiveCapture: "passed",
list: "passed", list: "passed",
snapshot: "passed", snapshot: "passed",
click: "passed", click: "passed",
@@ -446,10 +652,11 @@ async function main() {
client, client,
serverId: status.serverId, serverId: status.serverId,
targetUrl: target.url, targetUrl: target.url,
callerAgentId,
}); });
writeJson(path.join(artifactDir, "result.json"), report); writeJson(path.join(artifactDir, "result.json"), report);
console.log( console.log(
`Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} -> ${report.replacementWebContentsId}; list, snapshot, click passed.`, `Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click passed.`,
); );
} catch (error) { } catch (error) {
console.error(`Browser desktop browser E2E failed. Artifacts: ${artifactDir}`); console.error(`Browser desktop browser E2E failed. Artifacts: ${artifactDir}`);
@@ -20,11 +20,14 @@ class FakeDebugger {
public blockCommands = false; public blockCommands = false;
public readonly blockedCommandNames = new Set<string>(); public readonly blockedCommandNames = new Set<string>();
public readonly failedCommandNames = new Set<string>(); public readonly failedCommandNames = new Set<string>();
public readonly failedCommandErrors = new Map<string, Error>();
public readonly promptDialogs: unknown[] = []; public readonly promptDialogs: unknown[] = [];
public failPromptDrain = false; public failPromptDrain = false;
public beforeCommand: ((command: string) => void) | null = null;
private messageListener: private messageListener:
| ((event: unknown, method: string, params?: Record<string, unknown>) => void) | ((event: unknown, method: string, params?: Record<string, unknown>) => void)
| null = null; | null = null;
private detachListener: (() => void) | null = null;
private readonly blockedCommands: Array<() => void> = []; private readonly blockedCommands: Array<() => void> = [];
public isAttached(): boolean { public isAttached(): boolean {
@@ -37,6 +40,11 @@ class FakeDebugger {
public async sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown> { public async sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown> {
this.commands.push({ command, params: params ?? {} }); this.commands.push({ command, params: params ?? {} });
this.beforeCommand?.(command);
const commandError = this.failedCommandErrors.get(command);
if (commandError) {
throw commandError;
}
if (this.failedCommandNames.has(command)) { if (this.failedCommandNames.has(command)) {
throw new Error(`${command} failed`); throw new Error(`${command} failed`);
} }
@@ -58,10 +66,15 @@ class FakeDebugger {
} }
public on( public on(
event: "message", event: "message" | "detach",
listener: (event: unknown, method: string, params?: Record<string, unknown>) => void, listener:
| ((event: unknown, method: string, params?: Record<string, unknown>) => void)
| (() => void),
): void { ): void {
expect(event).toBe("message"); if (event === "detach") {
this.detachListener = listener;
return;
}
this.messageListener = listener; this.messageListener = listener;
} }
@@ -72,6 +85,11 @@ class FakeDebugger {
this.messageListener({}, method, params); this.messageListener({}, method, params);
} }
public emitDetach(): void {
this.attachedProtocolVersions.length = 0;
this.detachListener?.();
}
public finishNextCommand(): void { public finishNextCommand(): void {
const resolve = this.blockedCommands.shift(); const resolve = this.blockedCommands.shift();
if (!resolve) { if (!resolve) {
@@ -92,6 +110,7 @@ type ConsoleMessageListener = (
class FakeWebContents { class FakeWebContents {
public readonly debugger = new FakeDebugger(); public readonly debugger = new FakeDebugger();
public readonly inputEvents: IsolatedKeyboardInputEvent[] = []; public readonly inputEvents: IsolatedKeyboardInputEvent[] = [];
public readonly loadedUrls: string[] = [];
public readonly captures: Array<{ public readonly captures: Array<{
rect: Rectangle | undefined; rect: Rectangle | undefined;
options: { stayHidden?: boolean } | undefined; options: { stayHidden?: boolean } | undefined;
@@ -138,7 +157,9 @@ class FakeWebContents {
return null; return null;
} }
public async loadURL(): Promise<void> {} public async loadURL(url: string): Promise<void> {
this.loadedUrls.push(url);
}
public goBack(): void {} public goBack(): void {}
@@ -186,6 +207,7 @@ class FakeWebContents {
public destroy(): void { public destroy(): void {
this.destroyed = true; this.destroyed = true;
this.debugger.emitDetach();
this.destroyedListener?.(); this.destroyedListener?.();
} }
} }
@@ -571,6 +593,54 @@ describe("browser automation IPC adapter", () => {
); );
warn.mockRestore(); warn.mockRestore();
}); });
test("does not run the command after the debugger target closes during setup", async () => {
const contents = new FakeWebContents(31);
contents.debugger.beforeCommand = (command) => {
if (command === "Page.enable") {
contents.destroy();
}
};
contents.debugger.failedCommandErrors.set(
"Page.enable",
new Error("target closed while handling command"),
);
const tab = adaptWebContents(contents);
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
await expect(
tab.captureDialogs?.(() => tab.loadURL("https://replacement.example.com")),
).rejects.toThrow("target closed while handling command");
expect(contents.loadedUrls).toEqual([]);
expect(warn).not.toHaveBeenCalled();
warn.mockRestore();
});
test("runs without dialog capture after a live debugger session detaches", async () => {
const contents = new FakeWebContents(32);
const tab = adaptWebContents(contents);
await expect(tab.captureDialogs?.(async () => "captured")).resolves.toEqual({
result: "captured",
dialogs: [],
});
contents.debugger.emitDetach();
contents.debugger.failedCommandNames.add("Page.enable");
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
await expect(
tab.captureDialogs?.(() => tab.loadURL("https://replacement.example.com")),
).resolves.toEqual({ result: undefined, dialogs: [] });
expect(contents.loadedUrls).toEqual(["https://replacement.example.com"]);
expect(contents.debugger.attachedProtocolVersions).toEqual(["1.3"]);
expect(warn).toHaveBeenCalledWith(
"[browser-automation] Dialog capture unavailable; running command without it",
{ contentsId: 32, error: expect.any(Error) },
);
warn.mockRestore();
});
}); });
class FakeHostWebContents { class FakeHostWebContents {
@@ -74,6 +74,7 @@ interface WebContentsDebugger {
event: "message", event: "message",
listener: (event: unknown, method: string, params?: Record<string, unknown>) => void, listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
): void; ): void;
on?(event: "detach", listener: () => void): void;
} }
interface ConsoleMessageEmitter { interface ConsoleMessageEmitter {
@@ -188,6 +189,7 @@ function getDialogMonitor(
class DialogMonitor { class DialogMonitor {
private enabled = false; private enabled = false;
private listenerRegistered = false; private listenerRegistered = false;
private detachGeneration = 0;
private readonly activeCollectors: DialogCollector[] = []; private readonly activeCollectors: DialogCollector[] = [];
public constructor( public constructor(
@@ -200,10 +202,14 @@ class DialogMonitor {
task: () => Promise<T>, task: () => Promise<T>,
): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }> { ): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }> {
const collector: DialogCollector = { dialogs: [] }; const collector: DialogCollector = { dialogs: [] };
const setupDetachGeneration = this.detachGeneration;
try { try {
await this.enable(); await this.enable();
await this.installPromptShim(); await this.installPromptShim();
} catch (error) { } catch (error) {
if (this.contents.isDestroyed() || this.detachGeneration !== setupDetachGeneration) {
throw error;
}
console.warn("[browser-automation] Dialog capture unavailable; running command without it", { console.warn("[browser-automation] Dialog capture unavailable; running command without it", {
contentsId: this.contentsId, contentsId: this.contentsId,
error, error,
@@ -244,6 +250,10 @@ class DialogMonitor {
} }
void this.handleOpening(params ?? {}); void this.handleOpening(params ?? {});
}); });
this.contents.debugger.on("detach", () => {
this.enabled = false;
this.detachGeneration += 1;
});
} }
await this.sendDebugCommand("Page.enable"); await this.sendDebugCommand("Page.enable");
this.enabled = true; this.enabled = true;