fix(desktop): show browser element annotation editor

The editor was trapped below the resident webview's body-level paint plane in split layouts. Render it through the shared overlay root while tracking the owning pane's bounds.
This commit is contained in:
Mohamed Boudra
2026-08-17 21:15:15 +02:00
parent ac6a0afdc7
commit 7d3212e365
2 changed files with 105 additions and 6 deletions
@@ -1,6 +1,7 @@
import {
useCallback,
useEffect,
useLayoutEffect,
useMemo,
useRef,
useState,
@@ -8,6 +9,7 @@ import {
type ReactNode,
createElement,
} from "react";
import { createPortal } from "react-dom";
import { Pressable, Text, TextInput, View, type StyleProp, type ViewStyle } from "react-native";
import {
ArrowLeft,
@@ -45,6 +47,7 @@ import {
import type { AttachmentMetadata, BrowserElementAttachment } from "@/attachments/types";
import { persistAttachmentFromDataUrl } from "@/attachments/service";
import { WORKSPACE_SECONDARY_HEADER_HEIGHT } from "@/constants/layout";
import { getOverlayRoot } from "@/lib/overlay-root";
import {
getDesktopHost,
isElectronRuntime,
@@ -1540,6 +1543,7 @@ export function BrowserPane({
})}
{pendingSelection ? (
<BrowserElementAnnotationCard
anchor={webviewClipRef.current}
selection={pendingSelection}
onSubmit={submitAnnotation}
onCancel={cancelAnnotation}
@@ -1551,15 +1555,18 @@ export function BrowserPane({
}
function BrowserElementAnnotationCard({
anchor,
selection,
onSubmit,
onCancel,
}: {
anchor: HTMLElement | null;
selection: BrowserElementSelection;
onSubmit: (annotation: BrowserElementAnnotation) => void;
onCancel: () => void;
}) {
const { t } = useTranslation();
const bounds = useElementBounds(anchor);
const [comment, setComment] = useState("");
const commentRef = useRef(comment);
commentRef.current = comment;
@@ -1591,8 +1598,12 @@ function BrowserElementAnnotationCard({
const elementText = truncateText(selection.text.trim().replace(/\s+/g, " "), 60);
const elementLabel = elementText ? `${selection.tag} · ${elementText}` : selection.tag;
return (
<View style={styles.annotationOverlay} pointerEvents="box-none">
if (!bounds) {
return null;
}
return createPortal(
<View style={[styles.annotationOverlay, bounds]} pointerEvents="box-none">
<View style={styles.annotationCard}>
<View style={styles.annotationHeader}>
<Text numberOfLines={1} style={styles.annotationTitle}>
@@ -1629,10 +1640,36 @@ function BrowserElementAnnotationCard({
</Button>
</View>
</View>
</View>
</View>,
getOverlayRoot(),
);
}
function useElementBounds(element: HTMLElement | null): ViewStyle | null {
const [bounds, setBounds] = useState<ViewStyle | null>(null);
useLayoutEffect(() => {
if (!element) {
setBounds(null);
return;
}
const update = () => {
const rect = element.getBoundingClientRect();
setBounds({ left: rect.left, top: rect.top, width: rect.width, height: rect.height });
};
update();
const observer = new ResizeObserver(update);
observer.observe(element);
window.addEventListener("resize", update);
return () => {
observer.disconnect();
window.removeEventListener("resize", update);
};
}, [element]);
return bounds;
}
const ThemedCloseIcon = withUnistyles(X);
const ThemedAnnotationInput = withUnistyles(TextInput);
const iconForegroundMutedMapping = (theme: { colors: { foregroundMuted: string } }) => ({
@@ -1740,11 +1777,9 @@ const styles = StyleSheet.create((theme) => ({
annotationOverlay: {
position: "absolute",
zIndex: 1,
left: 0,
right: 0,
bottom: 0,
padding: theme.spacing[3],
alignItems: "center",
justifyContent: "flex-end",
},
annotationCard: {
width: "100%",
+64
View File
@@ -390,6 +390,41 @@ async function selectDeviceSize(page, label) {
return !openPixels.equals(closedPixels);
}
async function selectElementAndReadAnnotationPaint({ page, client, browserId, artifactDir }) {
await clickGuestElement(page, client, browserId, "#bridge-target");
const comment = page.getByRole("textbox", {
name: "Message to the agent about this element…",
});
await comment.waitFor({ state: "visible", timeout: timeoutMs });
const bounds = await comment.boundingBox();
assert(bounds, "Element annotation comment box had no bounds");
const receivesInput = await comment.evaluate((element) => {
const elementBounds = element.getBoundingClientRect();
const target = document.elementFromPoint(
elementBounds.left + elementBounds.width / 2,
elementBounds.top + elementBounds.height / 2,
);
return target === element || element.contains(target);
});
const clip = {
x: Math.max(0, bounds.x),
y: Math.max(0, bounds.y),
width: bounds.width,
height: bounds.height,
};
const openPixels = await page.screenshot({
clip,
path: path.join(artifactDir, "element-annotation-open.png"),
});
await page.keyboard.press("Escape");
await comment.waitFor({ state: "hidden", timeout: timeoutMs });
const closedPixels = await page.screenshot({
clip,
path: path.join(artifactDir, "element-annotation-closed.png"),
});
return receivesInput && !openPixels.equals(closedPixels);
}
function recordViewportMismatch(failures, label, actual, expected) {
if (actual.width === expected.width && actual.height === expected.height) {
return;
@@ -772,6 +807,35 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId,
await page.screenshot({ path: path.join(artifactDir, "local-page-screenshot-selector.png") });
await originalDeck.getByRole("button", { name: "Cancel element selector" }).click();
await originalDeck.getByTestId(`workspace-tab-agent_${callerAgentId}`).click();
await page.getByRole("button", { name: "Open command center" }).click();
await page.getByTestId("command-center-input").fill("Split pane right");
await page.getByText("Split pane right", { exact: true }).click();
assert(
(await originalDeck.getByTestId("workspace-tabs-row").filter({ visible: true }).count()) === 2,
"Split pane command did not produce two visible panes",
);
await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).last().click();
await originalDeck
.getByTestId(`browser-webview-clip-${browserId}`)
.waitFor({ state: "visible", timeout: timeoutMs });
const splitAnnotateButton = originalDeck.getByRole("button", { name: "Annotate element" });
await splitAnnotateButton.click();
assert(
await waitForGuestSelector(client, browserId),
"Element selector did not start in the split browser pane",
);
assert(
await selectElementAndReadAnnotationPaint({
page,
client,
browserId,
artifactDir,
}),
"Element annotation comment box did not paint or receive input above the browser surface",
);
if (failures.length > 0) {
throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`);
}