fix(desktop): show browser element annotation editor
The editor was trapped below the resident webview's body-level paint plane in split layouts. Render it through the shared overlay root while tracking the owning pane's bounds.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
useCallback,
|
||||
useEffect,
|
||||
useLayoutEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
@@ -8,6 +9,7 @@ import {
|
||||
type ReactNode,
|
||||
createElement,
|
||||
} from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import { Pressable, Text, TextInput, View, type StyleProp, type ViewStyle } from "react-native";
|
||||
import {
|
||||
ArrowLeft,
|
||||
@@ -45,6 +47,7 @@ import {
|
||||
import type { AttachmentMetadata, BrowserElementAttachment } from "@/attachments/types";
|
||||
import { persistAttachmentFromDataUrl } from "@/attachments/service";
|
||||
import { WORKSPACE_SECONDARY_HEADER_HEIGHT } from "@/constants/layout";
|
||||
import { getOverlayRoot } from "@/lib/overlay-root";
|
||||
import {
|
||||
getDesktopHost,
|
||||
isElectronRuntime,
|
||||
@@ -1540,6 +1543,7 @@ export function BrowserPane({
|
||||
})}
|
||||
{pendingSelection ? (
|
||||
<BrowserElementAnnotationCard
|
||||
anchor={webviewClipRef.current}
|
||||
selection={pendingSelection}
|
||||
onSubmit={submitAnnotation}
|
||||
onCancel={cancelAnnotation}
|
||||
@@ -1551,15 +1555,18 @@ export function BrowserPane({
|
||||
}
|
||||
|
||||
function BrowserElementAnnotationCard({
|
||||
anchor,
|
||||
selection,
|
||||
onSubmit,
|
||||
onCancel,
|
||||
}: {
|
||||
anchor: HTMLElement | null;
|
||||
selection: BrowserElementSelection;
|
||||
onSubmit: (annotation: BrowserElementAnnotation) => void;
|
||||
onCancel: () => void;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const bounds = useElementBounds(anchor);
|
||||
const [comment, setComment] = useState("");
|
||||
const commentRef = useRef(comment);
|
||||
commentRef.current = comment;
|
||||
@@ -1591,8 +1598,12 @@ function BrowserElementAnnotationCard({
|
||||
const elementText = truncateText(selection.text.trim().replace(/\s+/g, " "), 60);
|
||||
const elementLabel = elementText ? `${selection.tag} · ${elementText}` : selection.tag;
|
||||
|
||||
return (
|
||||
<View style={styles.annotationOverlay} pointerEvents="box-none">
|
||||
if (!bounds) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return createPortal(
|
||||
<View style={[styles.annotationOverlay, bounds]} pointerEvents="box-none">
|
||||
<View style={styles.annotationCard}>
|
||||
<View style={styles.annotationHeader}>
|
||||
<Text numberOfLines={1} style={styles.annotationTitle}>
|
||||
@@ -1629,10 +1640,36 @@ function BrowserElementAnnotationCard({
|
||||
</Button>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
</View>,
|
||||
getOverlayRoot(),
|
||||
);
|
||||
}
|
||||
|
||||
function useElementBounds(element: HTMLElement | null): ViewStyle | null {
|
||||
const [bounds, setBounds] = useState<ViewStyle | null>(null);
|
||||
|
||||
useLayoutEffect(() => {
|
||||
if (!element) {
|
||||
setBounds(null);
|
||||
return;
|
||||
}
|
||||
const update = () => {
|
||||
const rect = element.getBoundingClientRect();
|
||||
setBounds({ left: rect.left, top: rect.top, width: rect.width, height: rect.height });
|
||||
};
|
||||
update();
|
||||
const observer = new ResizeObserver(update);
|
||||
observer.observe(element);
|
||||
window.addEventListener("resize", update);
|
||||
return () => {
|
||||
observer.disconnect();
|
||||
window.removeEventListener("resize", update);
|
||||
};
|
||||
}, [element]);
|
||||
|
||||
return bounds;
|
||||
}
|
||||
|
||||
const ThemedCloseIcon = withUnistyles(X);
|
||||
const ThemedAnnotationInput = withUnistyles(TextInput);
|
||||
const iconForegroundMutedMapping = (theme: { colors: { foregroundMuted: string } }) => ({
|
||||
@@ -1740,11 +1777,9 @@ const styles = StyleSheet.create((theme) => ({
|
||||
annotationOverlay: {
|
||||
position: "absolute",
|
||||
zIndex: 1,
|
||||
left: 0,
|
||||
right: 0,
|
||||
bottom: 0,
|
||||
padding: theme.spacing[3],
|
||||
alignItems: "center",
|
||||
justifyContent: "flex-end",
|
||||
},
|
||||
annotationCard: {
|
||||
width: "100%",
|
||||
|
||||
@@ -390,6 +390,41 @@ async function selectDeviceSize(page, label) {
|
||||
return !openPixels.equals(closedPixels);
|
||||
}
|
||||
|
||||
async function selectElementAndReadAnnotationPaint({ page, client, browserId, artifactDir }) {
|
||||
await clickGuestElement(page, client, browserId, "#bridge-target");
|
||||
const comment = page.getByRole("textbox", {
|
||||
name: "Message to the agent about this element…",
|
||||
});
|
||||
await comment.waitFor({ state: "visible", timeout: timeoutMs });
|
||||
const bounds = await comment.boundingBox();
|
||||
assert(bounds, "Element annotation comment box had no bounds");
|
||||
const receivesInput = await comment.evaluate((element) => {
|
||||
const elementBounds = element.getBoundingClientRect();
|
||||
const target = document.elementFromPoint(
|
||||
elementBounds.left + elementBounds.width / 2,
|
||||
elementBounds.top + elementBounds.height / 2,
|
||||
);
|
||||
return target === element || element.contains(target);
|
||||
});
|
||||
const clip = {
|
||||
x: Math.max(0, bounds.x),
|
||||
y: Math.max(0, bounds.y),
|
||||
width: bounds.width,
|
||||
height: bounds.height,
|
||||
};
|
||||
const openPixels = await page.screenshot({
|
||||
clip,
|
||||
path: path.join(artifactDir, "element-annotation-open.png"),
|
||||
});
|
||||
await page.keyboard.press("Escape");
|
||||
await comment.waitFor({ state: "hidden", timeout: timeoutMs });
|
||||
const closedPixels = await page.screenshot({
|
||||
clip,
|
||||
path: path.join(artifactDir, "element-annotation-closed.png"),
|
||||
});
|
||||
return receivesInput && !openPixels.equals(closedPixels);
|
||||
}
|
||||
|
||||
function recordViewportMismatch(failures, label, actual, expected) {
|
||||
if (actual.width === expected.width && actual.height === expected.height) {
|
||||
return;
|
||||
@@ -772,6 +807,35 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId,
|
||||
await page.screenshot({ path: path.join(artifactDir, "local-page-screenshot-selector.png") });
|
||||
await originalDeck.getByRole("button", { name: "Cancel element selector" }).click();
|
||||
|
||||
await originalDeck.getByTestId(`workspace-tab-agent_${callerAgentId}`).click();
|
||||
await page.getByRole("button", { name: "Open command center" }).click();
|
||||
await page.getByTestId("command-center-input").fill("Split pane right");
|
||||
await page.getByText("Split pane right", { exact: true }).click();
|
||||
assert(
|
||||
(await originalDeck.getByTestId("workspace-tabs-row").filter({ visible: true }).count()) === 2,
|
||||
"Split pane command did not produce two visible panes",
|
||||
);
|
||||
await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).last().click();
|
||||
await originalDeck
|
||||
.getByTestId(`browser-webview-clip-${browserId}`)
|
||||
.waitFor({ state: "visible", timeout: timeoutMs });
|
||||
|
||||
const splitAnnotateButton = originalDeck.getByRole("button", { name: "Annotate element" });
|
||||
await splitAnnotateButton.click();
|
||||
assert(
|
||||
await waitForGuestSelector(client, browserId),
|
||||
"Element selector did not start in the split browser pane",
|
||||
);
|
||||
assert(
|
||||
await selectElementAndReadAnnotationPaint({
|
||||
page,
|
||||
client,
|
||||
browserId,
|
||||
artifactDir,
|
||||
}),
|
||||
"Element annotation comment box did not paint or receive input above the browser surface",
|
||||
);
|
||||
|
||||
if (failures.length > 0) {
|
||||
throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user