From 7d3212e36588ec6ea98a819f171f9ca87f15b7cf Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 17 Aug 2026 20:28:14 +0200 Subject: [PATCH] fix(desktop): show browser element annotation editor The editor was trapped below the resident webview's body-level paint plane in split layouts. Render it through the shared overlay root while tracking the owning pane's bounds. --- .../desktop/browser/pane/index.electron.tsx | 47 ++++++++++++-- packages/desktop/e2e/browser-tabs.e2e.mjs | 64 +++++++++++++++++++ 2 files changed, 105 insertions(+), 6 deletions(-) diff --git a/packages/app/src/desktop/browser/pane/index.electron.tsx b/packages/app/src/desktop/browser/pane/index.electron.tsx index 4edc016ee..589aa0ad8 100644 --- a/packages/app/src/desktop/browser/pane/index.electron.tsx +++ b/packages/app/src/desktop/browser/pane/index.electron.tsx @@ -1,6 +1,7 @@ import { useCallback, useEffect, + useLayoutEffect, useMemo, useRef, useState, @@ -8,6 +9,7 @@ import { type ReactNode, createElement, } from "react"; +import { createPortal } from "react-dom"; import { Pressable, Text, TextInput, View, type StyleProp, type ViewStyle } from "react-native"; import { ArrowLeft, @@ -45,6 +47,7 @@ import { import type { AttachmentMetadata, BrowserElementAttachment } from "@/attachments/types"; import { persistAttachmentFromDataUrl } from "@/attachments/service"; import { WORKSPACE_SECONDARY_HEADER_HEIGHT } from "@/constants/layout"; +import { getOverlayRoot } from "@/lib/overlay-root"; import { getDesktopHost, isElectronRuntime, @@ -1540,6 +1543,7 @@ export function BrowserPane({ })} {pendingSelection ? ( void; onCancel: () => void; }) { const { t } = useTranslation(); + const bounds = useElementBounds(anchor); const [comment, setComment] = useState(""); const commentRef = useRef(comment); commentRef.current = comment; @@ -1591,8 +1598,12 @@ function BrowserElementAnnotationCard({ const elementText = truncateText(selection.text.trim().replace(/\s+/g, " "), 60); const elementLabel = elementText ? `${selection.tag} · ${elementText}` : selection.tag; - return ( - + if (!bounds) { + return null; + } + + return createPortal( + @@ -1629,10 +1640,36 @@ function BrowserElementAnnotationCard({ - + , + getOverlayRoot(), ); } +function useElementBounds(element: HTMLElement | null): ViewStyle | null { + const [bounds, setBounds] = useState(null); + + useLayoutEffect(() => { + if (!element) { + setBounds(null); + return; + } + const update = () => { + const rect = element.getBoundingClientRect(); + setBounds({ left: rect.left, top: rect.top, width: rect.width, height: rect.height }); + }; + update(); + const observer = new ResizeObserver(update); + observer.observe(element); + window.addEventListener("resize", update); + return () => { + observer.disconnect(); + window.removeEventListener("resize", update); + }; + }, [element]); + + return bounds; +} + const ThemedCloseIcon = withUnistyles(X); const ThemedAnnotationInput = withUnistyles(TextInput); const iconForegroundMutedMapping = (theme: { colors: { foregroundMuted: string } }) => ({ @@ -1740,11 +1777,9 @@ const styles = StyleSheet.create((theme) => ({ annotationOverlay: { position: "absolute", zIndex: 1, - left: 0, - right: 0, - bottom: 0, padding: theme.spacing[3], alignItems: "center", + justifyContent: "flex-end", }, annotationCard: { width: "100%", diff --git a/packages/desktop/e2e/browser-tabs.e2e.mjs b/packages/desktop/e2e/browser-tabs.e2e.mjs index 7ff2bee92..30eeb5efc 100644 --- a/packages/desktop/e2e/browser-tabs.e2e.mjs +++ b/packages/desktop/e2e/browser-tabs.e2e.mjs @@ -390,6 +390,41 @@ async function selectDeviceSize(page, label) { return !openPixels.equals(closedPixels); } +async function selectElementAndReadAnnotationPaint({ page, client, browserId, artifactDir }) { + await clickGuestElement(page, client, browserId, "#bridge-target"); + const comment = page.getByRole("textbox", { + name: "Message to the agent about this element…", + }); + await comment.waitFor({ state: "visible", timeout: timeoutMs }); + const bounds = await comment.boundingBox(); + assert(bounds, "Element annotation comment box had no bounds"); + const receivesInput = await comment.evaluate((element) => { + const elementBounds = element.getBoundingClientRect(); + const target = document.elementFromPoint( + elementBounds.left + elementBounds.width / 2, + elementBounds.top + elementBounds.height / 2, + ); + return target === element || element.contains(target); + }); + const clip = { + x: Math.max(0, bounds.x), + y: Math.max(0, bounds.y), + width: bounds.width, + height: bounds.height, + }; + const openPixels = await page.screenshot({ + clip, + path: path.join(artifactDir, "element-annotation-open.png"), + }); + await page.keyboard.press("Escape"); + await comment.waitFor({ state: "hidden", timeout: timeoutMs }); + const closedPixels = await page.screenshot({ + clip, + path: path.join(artifactDir, "element-annotation-closed.png"), + }); + return receivesInput && !openPixels.equals(closedPixels); +} + function recordViewportMismatch(failures, label, actual, expected) { if (actual.width === expected.width && actual.height === expected.height) { return; @@ -772,6 +807,35 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId, await page.screenshot({ path: path.join(artifactDir, "local-page-screenshot-selector.png") }); await originalDeck.getByRole("button", { name: "Cancel element selector" }).click(); + await originalDeck.getByTestId(`workspace-tab-agent_${callerAgentId}`).click(); + await page.getByRole("button", { name: "Open command center" }).click(); + await page.getByTestId("command-center-input").fill("Split pane right"); + await page.getByText("Split pane right", { exact: true }).click(); + assert( + (await originalDeck.getByTestId("workspace-tabs-row").filter({ visible: true }).count()) === 2, + "Split pane command did not produce two visible panes", + ); + await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).last().click(); + await originalDeck + .getByTestId(`browser-webview-clip-${browserId}`) + .waitFor({ state: "visible", timeout: timeoutMs }); + + const splitAnnotateButton = originalDeck.getByRole("button", { name: "Annotate element" }); + await splitAnnotateButton.click(); + assert( + await waitForGuestSelector(client, browserId), + "Element selector did not start in the split browser pane", + ); + assert( + await selectElementAndReadAnnotationPaint({ + page, + client, + browserId, + artifactDir, + }), + "Element annotation comment box did not paint or receive input above the browser surface", + ); + if (failures.length > 0) { throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`); }