fix(desktop): show browser element annotation editor
The editor was trapped below the resident webview's body-level paint plane in split layouts. Render it through the shared overlay root while tracking the owning pane's bounds.
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
import {
|
import {
|
||||||
useCallback,
|
useCallback,
|
||||||
useEffect,
|
useEffect,
|
||||||
|
useLayoutEffect,
|
||||||
useMemo,
|
useMemo,
|
||||||
useRef,
|
useRef,
|
||||||
useState,
|
useState,
|
||||||
@@ -8,6 +9,7 @@ import {
|
|||||||
type ReactNode,
|
type ReactNode,
|
||||||
createElement,
|
createElement,
|
||||||
} from "react";
|
} from "react";
|
||||||
|
import { createPortal } from "react-dom";
|
||||||
import { Pressable, Text, TextInput, View, type StyleProp, type ViewStyle } from "react-native";
|
import { Pressable, Text, TextInput, View, type StyleProp, type ViewStyle } from "react-native";
|
||||||
import {
|
import {
|
||||||
ArrowLeft,
|
ArrowLeft,
|
||||||
@@ -45,6 +47,7 @@ import {
|
|||||||
import type { AttachmentMetadata, BrowserElementAttachment } from "@/attachments/types";
|
import type { AttachmentMetadata, BrowserElementAttachment } from "@/attachments/types";
|
||||||
import { persistAttachmentFromDataUrl } from "@/attachments/service";
|
import { persistAttachmentFromDataUrl } from "@/attachments/service";
|
||||||
import { WORKSPACE_SECONDARY_HEADER_HEIGHT } from "@/constants/layout";
|
import { WORKSPACE_SECONDARY_HEADER_HEIGHT } from "@/constants/layout";
|
||||||
|
import { getOverlayRoot } from "@/lib/overlay-root";
|
||||||
import {
|
import {
|
||||||
getDesktopHost,
|
getDesktopHost,
|
||||||
isElectronRuntime,
|
isElectronRuntime,
|
||||||
@@ -1540,6 +1543,7 @@ export function BrowserPane({
|
|||||||
})}
|
})}
|
||||||
{pendingSelection ? (
|
{pendingSelection ? (
|
||||||
<BrowserElementAnnotationCard
|
<BrowserElementAnnotationCard
|
||||||
|
anchor={webviewClipRef.current}
|
||||||
selection={pendingSelection}
|
selection={pendingSelection}
|
||||||
onSubmit={submitAnnotation}
|
onSubmit={submitAnnotation}
|
||||||
onCancel={cancelAnnotation}
|
onCancel={cancelAnnotation}
|
||||||
@@ -1551,15 +1555,18 @@ export function BrowserPane({
|
|||||||
}
|
}
|
||||||
|
|
||||||
function BrowserElementAnnotationCard({
|
function BrowserElementAnnotationCard({
|
||||||
|
anchor,
|
||||||
selection,
|
selection,
|
||||||
onSubmit,
|
onSubmit,
|
||||||
onCancel,
|
onCancel,
|
||||||
}: {
|
}: {
|
||||||
|
anchor: HTMLElement | null;
|
||||||
selection: BrowserElementSelection;
|
selection: BrowserElementSelection;
|
||||||
onSubmit: (annotation: BrowserElementAnnotation) => void;
|
onSubmit: (annotation: BrowserElementAnnotation) => void;
|
||||||
onCancel: () => void;
|
onCancel: () => void;
|
||||||
}) {
|
}) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
|
const bounds = useElementBounds(anchor);
|
||||||
const [comment, setComment] = useState("");
|
const [comment, setComment] = useState("");
|
||||||
const commentRef = useRef(comment);
|
const commentRef = useRef(comment);
|
||||||
commentRef.current = comment;
|
commentRef.current = comment;
|
||||||
@@ -1591,8 +1598,12 @@ function BrowserElementAnnotationCard({
|
|||||||
const elementText = truncateText(selection.text.trim().replace(/\s+/g, " "), 60);
|
const elementText = truncateText(selection.text.trim().replace(/\s+/g, " "), 60);
|
||||||
const elementLabel = elementText ? `${selection.tag} · ${elementText}` : selection.tag;
|
const elementLabel = elementText ? `${selection.tag} · ${elementText}` : selection.tag;
|
||||||
|
|
||||||
return (
|
if (!bounds) {
|
||||||
<View style={styles.annotationOverlay} pointerEvents="box-none">
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return createPortal(
|
||||||
|
<View style={[styles.annotationOverlay, bounds]} pointerEvents="box-none">
|
||||||
<View style={styles.annotationCard}>
|
<View style={styles.annotationCard}>
|
||||||
<View style={styles.annotationHeader}>
|
<View style={styles.annotationHeader}>
|
||||||
<Text numberOfLines={1} style={styles.annotationTitle}>
|
<Text numberOfLines={1} style={styles.annotationTitle}>
|
||||||
@@ -1629,10 +1640,36 @@ function BrowserElementAnnotationCard({
|
|||||||
</Button>
|
</Button>
|
||||||
</View>
|
</View>
|
||||||
</View>
|
</View>
|
||||||
</View>
|
</View>,
|
||||||
|
getOverlayRoot(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function useElementBounds(element: HTMLElement | null): ViewStyle | null {
|
||||||
|
const [bounds, setBounds] = useState<ViewStyle | null>(null);
|
||||||
|
|
||||||
|
useLayoutEffect(() => {
|
||||||
|
if (!element) {
|
||||||
|
setBounds(null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const update = () => {
|
||||||
|
const rect = element.getBoundingClientRect();
|
||||||
|
setBounds({ left: rect.left, top: rect.top, width: rect.width, height: rect.height });
|
||||||
|
};
|
||||||
|
update();
|
||||||
|
const observer = new ResizeObserver(update);
|
||||||
|
observer.observe(element);
|
||||||
|
window.addEventListener("resize", update);
|
||||||
|
return () => {
|
||||||
|
observer.disconnect();
|
||||||
|
window.removeEventListener("resize", update);
|
||||||
|
};
|
||||||
|
}, [element]);
|
||||||
|
|
||||||
|
return bounds;
|
||||||
|
}
|
||||||
|
|
||||||
const ThemedCloseIcon = withUnistyles(X);
|
const ThemedCloseIcon = withUnistyles(X);
|
||||||
const ThemedAnnotationInput = withUnistyles(TextInput);
|
const ThemedAnnotationInput = withUnistyles(TextInput);
|
||||||
const iconForegroundMutedMapping = (theme: { colors: { foregroundMuted: string } }) => ({
|
const iconForegroundMutedMapping = (theme: { colors: { foregroundMuted: string } }) => ({
|
||||||
@@ -1740,11 +1777,9 @@ const styles = StyleSheet.create((theme) => ({
|
|||||||
annotationOverlay: {
|
annotationOverlay: {
|
||||||
position: "absolute",
|
position: "absolute",
|
||||||
zIndex: 1,
|
zIndex: 1,
|
||||||
left: 0,
|
|
||||||
right: 0,
|
|
||||||
bottom: 0,
|
|
||||||
padding: theme.spacing[3],
|
padding: theme.spacing[3],
|
||||||
alignItems: "center",
|
alignItems: "center",
|
||||||
|
justifyContent: "flex-end",
|
||||||
},
|
},
|
||||||
annotationCard: {
|
annotationCard: {
|
||||||
width: "100%",
|
width: "100%",
|
||||||
|
|||||||
@@ -390,6 +390,41 @@ async function selectDeviceSize(page, label) {
|
|||||||
return !openPixels.equals(closedPixels);
|
return !openPixels.equals(closedPixels);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function selectElementAndReadAnnotationPaint({ page, client, browserId, artifactDir }) {
|
||||||
|
await clickGuestElement(page, client, browserId, "#bridge-target");
|
||||||
|
const comment = page.getByRole("textbox", {
|
||||||
|
name: "Message to the agent about this element…",
|
||||||
|
});
|
||||||
|
await comment.waitFor({ state: "visible", timeout: timeoutMs });
|
||||||
|
const bounds = await comment.boundingBox();
|
||||||
|
assert(bounds, "Element annotation comment box had no bounds");
|
||||||
|
const receivesInput = await comment.evaluate((element) => {
|
||||||
|
const elementBounds = element.getBoundingClientRect();
|
||||||
|
const target = document.elementFromPoint(
|
||||||
|
elementBounds.left + elementBounds.width / 2,
|
||||||
|
elementBounds.top + elementBounds.height / 2,
|
||||||
|
);
|
||||||
|
return target === element || element.contains(target);
|
||||||
|
});
|
||||||
|
const clip = {
|
||||||
|
x: Math.max(0, bounds.x),
|
||||||
|
y: Math.max(0, bounds.y),
|
||||||
|
width: bounds.width,
|
||||||
|
height: bounds.height,
|
||||||
|
};
|
||||||
|
const openPixels = await page.screenshot({
|
||||||
|
clip,
|
||||||
|
path: path.join(artifactDir, "element-annotation-open.png"),
|
||||||
|
});
|
||||||
|
await page.keyboard.press("Escape");
|
||||||
|
await comment.waitFor({ state: "hidden", timeout: timeoutMs });
|
||||||
|
const closedPixels = await page.screenshot({
|
||||||
|
clip,
|
||||||
|
path: path.join(artifactDir, "element-annotation-closed.png"),
|
||||||
|
});
|
||||||
|
return receivesInput && !openPixels.equals(closedPixels);
|
||||||
|
}
|
||||||
|
|
||||||
function recordViewportMismatch(failures, label, actual, expected) {
|
function recordViewportMismatch(failures, label, actual, expected) {
|
||||||
if (actual.width === expected.width && actual.height === expected.height) {
|
if (actual.width === expected.width && actual.height === expected.height) {
|
||||||
return;
|
return;
|
||||||
@@ -772,6 +807,35 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId,
|
|||||||
await page.screenshot({ path: path.join(artifactDir, "local-page-screenshot-selector.png") });
|
await page.screenshot({ path: path.join(artifactDir, "local-page-screenshot-selector.png") });
|
||||||
await originalDeck.getByRole("button", { name: "Cancel element selector" }).click();
|
await originalDeck.getByRole("button", { name: "Cancel element selector" }).click();
|
||||||
|
|
||||||
|
await originalDeck.getByTestId(`workspace-tab-agent_${callerAgentId}`).click();
|
||||||
|
await page.getByRole("button", { name: "Open command center" }).click();
|
||||||
|
await page.getByTestId("command-center-input").fill("Split pane right");
|
||||||
|
await page.getByText("Split pane right", { exact: true }).click();
|
||||||
|
assert(
|
||||||
|
(await originalDeck.getByTestId("workspace-tabs-row").filter({ visible: true }).count()) === 2,
|
||||||
|
"Split pane command did not produce two visible panes",
|
||||||
|
);
|
||||||
|
await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).last().click();
|
||||||
|
await originalDeck
|
||||||
|
.getByTestId(`browser-webview-clip-${browserId}`)
|
||||||
|
.waitFor({ state: "visible", timeout: timeoutMs });
|
||||||
|
|
||||||
|
const splitAnnotateButton = originalDeck.getByRole("button", { name: "Annotate element" });
|
||||||
|
await splitAnnotateButton.click();
|
||||||
|
assert(
|
||||||
|
await waitForGuestSelector(client, browserId),
|
||||||
|
"Element selector did not start in the split browser pane",
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
await selectElementAndReadAnnotationPaint({
|
||||||
|
page,
|
||||||
|
client,
|
||||||
|
browserId,
|
||||||
|
artifactDir,
|
||||||
|
}),
|
||||||
|
"Element annotation comment box did not paint or receive input above the browser surface",
|
||||||
|
);
|
||||||
|
|
||||||
if (failures.length > 0) {
|
if (failures.length > 0) {
|
||||||
throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`);
|
throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user