fix(desktop): keep element selectors available on loaded pages (#3187)

* fix(desktop): keep element selectors available on loaded pages

Annotate element and Screenshot element returned early when a cached load signal said the guest was not ready, even if the current document was complete and executable. Let executeJavaScript own the readiness boundary and keep selector cleanup available in the same stale-signal state.

Extend the real Electron browser regression to reproduce the complete-document/stale-signal mismatch through both toolbar actions.

* test(desktop): await browser selector injection

Poll the guest selector state through the browser bridge instead of assuming injection completes within 250 ms. This keeps the Electron regression deterministic on slower CI hosts.

* Harden browser selector session lifecycle

* Keep selector E2E fixtures independent

* fix(desktop): harden selector and annotation lifecycle

* fix(desktop): keep selector PR focused

* refactor(desktop): own element selector lifecycle

---------

Co-authored-by: Daniel G. Kang <180234654+dgk-dev@users.noreply.github.com>
Co-authored-by: Mohamed Boudra <boudra.moha@gmail.com>
This commit is contained in:
Daniel G. Kang
2026-08-14 15:51:42 +08:00
committed by GitHub
co-authored by Daniel G. Kang Mohamed Boudra
parent b03dc804eb
commit 770b87a15c
14 changed files with 649 additions and 362 deletions
@@ -0,0 +1,470 @@
import type { BrowserElementAttachment } from "@/attachments/types";
export type BrowserElementSelection = Omit<BrowserElementAttachment, "formatted" | "comment"> & {
attributes?: Record<string, string>;
};
export type ElementSelectorMode = "annotate" | "screenshot";
export type ElementSelectorOutcome =
| { type: "selected"; mode: ElementSelectorMode; selection: BrowserElementSelection }
| { type: "cancelled" }
| { type: "failed"; reason: "loading" | "timeout" | "unavailable" };
export type ElementSelectorStartResult = "started" | "loading" | "unavailable";
interface ElementSelectorWebview extends HTMLElement {
isLoading?: () => boolean;
executeJavaScript?: (code: string) => Promise<unknown>;
}
interface ElementSelectorSession {
mode: ElementSelectorMode;
token: string;
webview: ElementSelectorWebview;
onFinish: (outcome: ElementSelectorOutcome) => void;
stopPolling?: () => void;
timeoutId?: number;
}
export interface ElementSelectorController {
start(input: {
webview: ElementSelectorWebview;
mode: ElementSelectorMode;
onFinish: (outcome: ElementSelectorOutcome) => void;
}): ElementSelectorStartResult;
cancel(): void;
stopForWebview(webview: ElementSelectorWebview): void;
}
const SELECTOR_TIMEOUT_MS = 30_000;
function executeWebviewJavaScript(webview: ElementSelectorWebview, code: string): Promise<unknown> {
if (!webview.isConnected) {
return Promise.resolve(null);
}
try {
return webview.executeJavaScript?.(code) ?? Promise.resolve(null);
} catch (error) {
return Promise.reject(error);
}
}
function ignoreWebviewJavaScriptError() {}
function destroyWebviewSelector(webview: ElementSelectorWebview, sessionToken: string): void {
const token = JSON.stringify(sessionToken);
void executeWebviewJavaScript(
webview,
`if (window.__paseoSelector?.sessionToken === ${token}) window.__paseoSelector.destroy();`,
).catch(ignoreWebviewJavaScriptError);
}
function clearWebviewSelector(webview: ElementSelectorWebview, sessionToken: string): void {
const token = JSON.stringify(sessionToken);
void executeWebviewJavaScript(
webview,
`if (window.__paseoSelector?.sessionToken === ${token}) window.__paseoSelector.destroy(); if (window.__paseoSelectorResult?.__paseoSessionToken === ${token}) window.__paseoSelectorResult = null;`,
).catch(ignoreWebviewJavaScriptError);
}
function readSelectorInstallation(
value: unknown,
sessionToken: string,
): "installed" | "loading" | "unavailable" {
if (value === null || typeof value !== "object") {
return "unavailable";
}
if (Reflect.get(value, "sessionToken") !== sessionToken) {
return "unavailable";
}
if (Reflect.get(value, "installed") === true) {
return "installed";
}
return Reflect.get(value, "reason") === "document-loading" ? "loading" : "unavailable";
}
function stopSessionTimers(session: ElementSelectorSession): void {
session.stopPolling?.();
session.stopPolling = undefined;
if (session.timeoutId !== undefined) {
window.clearTimeout(session.timeoutId);
session.timeoutId = undefined;
}
}
function startSelectorResultPolling(input: {
webview: ElementSelectorWebview;
sessionToken: string;
onResult: (selection: BrowserElementSelection | null) => void;
}): () => void {
const { webview, sessionToken, onResult } = input;
const token = JSON.stringify(sessionToken);
let stopped = false;
let timerId: number | undefined;
const schedule = () => {
if (!stopped) {
timerId = window.setTimeout(poll, 200);
}
};
const poll = () => {
void (async () => {
try {
const raw = await executeWebviewJavaScript(
webview,
`JSON.stringify(window.__paseoSelectorResult?.__paseoSessionToken === ${token} ? window.__paseoSelectorResult : null)`,
);
const result = typeof raw === "string" ? JSON.parse(raw) : null;
if (!result) {
schedule();
return;
}
stopped = true;
await executeWebviewJavaScript(
webview,
`if (window.__paseoSelectorResult?.__paseoSessionToken === ${token}) window.__paseoSelectorResult = null;`,
).catch(ignoreWebviewJavaScriptError);
const cancelled = result.__cancelled === true;
delete result.__cancelled;
delete result.__paseoSessionToken;
onResult(cancelled ? null : (result as BrowserElementSelection));
} catch {
schedule();
}
})();
};
schedule();
return () => {
stopped = true;
if (timerId !== undefined) {
window.clearTimeout(timerId);
}
};
}
function buildElementSelectorScript(sessionToken: string): string {
const token = JSON.stringify(sessionToken);
return `
(function() {
var sessionToken = ${token};
if (document.readyState === 'loading' || !document.head || !document.documentElement) {
return { installed: false, reason: 'document-loading', sessionToken: sessionToken };
}
if (window.__paseoSelector) { window.__paseoSelector.destroy(); }
window.__paseoSelectorResult = null;
var style = document.createElement('style');
style.textContent = [
'.__paseo-hover { outline: 2px solid #3b82f6 !important; outline-offset: 2px !important; cursor: crosshair !important; }',
'.__paseo-select-mode, .__paseo-select-mode * { cursor: crosshair !important; pointer-events: auto !important; user-select: none !important; }',
'.__paseo-select-mode *, .__paseo-select-mode *::before, .__paseo-select-mode *::after { animation: none !important; transition: none !important; }',
'.__paseo-select-mode a, .__paseo-select-mode button, .__paseo-select-mode input, .__paseo-select-mode select, .__paseo-select-mode textarea, .__paseo-select-mode [role="button"], .__paseo-select-mode [onclick] { pointer-events: none !important; }',
'.__paseo-select-mode iframe, .__paseo-select-mode video, .__paseo-select-mode audio { pointer-events: none !important; }',
'.__paseo-hover-label { position: fixed; z-index: 2147483647; pointer-events: none; max-width: 360px; padding: 4px 8px; border-radius: 6px; background: rgba(24,24,27,0.96); color: #fff; font: 500 11px/1.45 ui-monospace,SFMono-Regular,Menlo,monospace; box-shadow: 0 2px 10px rgba(0,0,0,0.35); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }',
'.__paseo-hover-label .__paseo-tag { color: #93c5fd; }',
'.__paseo-hover-label .__paseo-id { color: #fca5a5; }',
'.__paseo-hover-label .__paseo-cls { color: #fcd34d; }',
'.__paseo-hover-label .__paseo-dim { color: #a1a1aa; margin-left: 6px; }',
'.__paseo-hover-label .__paseo-comp { color: #86efac; margin-left: 6px; }',
].join('\\n');
document.head.appendChild(style);
document.documentElement.classList.add('__paseo-select-mode');
var hoverLabel = document.createElement('div');
hoverLabel.className = '__paseo-hover-label';
hoverLabel.style.display = 'none';
document.documentElement.appendChild(hoverLabel);
var last = null;
function escapeHtml(value) {
return String(value).replace(/[&<>"]/g, function(ch) {
return ch === '&' ? '&amp;' : ch === '<' ? '&lt;' : ch === '>' ? '&gt;' : '&quot;';
});
}
function describeElement(el) {
var tag = el.tagName ? el.tagName.toLowerCase() : 'node';
var parts = ['<span class="__paseo-tag">' + escapeHtml(tag) + '</span>'];
if (el.id) {
parts.push('<span class="__paseo-id">#' + escapeHtml(el.id) + '</span>');
}
if (el.classList && el.classList.length) {
var cls = Array.prototype.slice.call(el.classList, 0, 2)
.filter(function(c) { return c.indexOf('__paseo') !== 0; })
.map(function(c) { return '.' + escapeHtml(c); })
.join('');
if (cls) parts.push('<span class="__paseo-cls">' + cls + '</span>');
}
var comp = getReactSource(el);
if (comp && comp.componentName) {
parts.push('<span class="__paseo-comp">&lt;' + escapeHtml(comp.componentName) + '&gt;</span>');
}
var rect = el.getBoundingClientRect();
parts.push('<span class="__paseo-dim">' + Math.round(rect.width) + '×' + Math.round(rect.height) + '</span>');
return { html: parts.join(''), rect: rect };
}
function positionLabel(rect, e) {
var lw = hoverLabel.offsetWidth || 0;
var lh = hoverLabel.offsetHeight || 0;
var top = rect.top - lh - 6;
if (top < 4) top = rect.bottom + 6;
if (top + lh > window.innerHeight - 4) top = Math.max(4, e.clientY - lh - 6);
var left = rect.left;
if (left + lw > window.innerWidth - 4) left = Math.max(4, window.innerWidth - lw - 4);
if (left < 4) left = 4;
hoverLabel.style.top = Math.round(top) + 'px';
hoverLabel.style.left = Math.round(left) + 'px';
}
function onMove(e) {
e.preventDefault();
e.stopPropagation();
if (last) last.classList.remove('__paseo-hover');
var el = e.target;
el.classList.add('__paseo-hover');
last = el;
try {
var info = describeElement(el);
hoverLabel.innerHTML = info.html;
hoverLabel.style.display = 'block';
positionLabel(info.rect, e);
} catch (err) {
hoverLabel.style.display = 'none';
}
}
function buildSelector(el) {
if (el.id) return '#' + el.id;
var path = [];
while (el && el.nodeType === 1) {
var seg = el.tagName.toLowerCase();
if (el.id) { path.unshift('#' + el.id); break; }
var sib = el, nth = 1;
while (sib = sib.previousElementSibling) { if (sib.tagName === el.tagName) nth++; }
if (nth > 1) seg += ':nth-of-type(' + nth + ')';
path.unshift(seg);
el = el.parentElement;
}
return path.join(' > ');
}
function getReactSource(el) {
var keys = Object.keys(el);
for (var i = 0; i < keys.length; i++) {
if (keys[i].startsWith('__reactFiber$') || keys[i].startsWith('__reactInternalInstance$')) {
var fiber = el[keys[i]];
while (fiber) {
if (fiber._debugSource) {
return {
fileName: fiber._debugSource.fileName || null,
lineNumber: fiber._debugSource.lineNumber || null,
columnNumber: fiber._debugSource.columnNumber || null,
componentName: (fiber.type && (typeof fiber.type === 'string' ? fiber.type : fiber.type.displayName || fiber.type.name)) || null
};
}
if (fiber._debugOwner) { fiber = fiber._debugOwner; }
else if (fiber.return) { fiber = fiber.return; }
else break;
}
}
}
return null;
}
function getParentChain(el, depth) {
var chain = [];
var cur = el.parentElement;
for (var i = 0; i < (depth || 5) && cur; i++) {
var desc = cur.tagName.toLowerCase();
if (cur.id) desc += '#' + cur.id;
if (cur.className && typeof cur.className === 'string') { var cls = cur.className.trim().replace(/ +/g, ' ').split(' ').slice(0,2).join('.'); if (cls) desc += '.' + cls; }
chain.push(desc);
cur = cur.parentElement;
}
return chain;
}
function getChildSummary(el, max) {
var kids = [];
for (var i = 0; i < Math.min(el.children.length, max || 8); i++) {
var c = el.children[i];
var desc = c.tagName.toLowerCase();
if (c.id) desc += '#' + c.id;
kids.push(desc);
}
if (el.children.length > (max || 8)) kids.push('...(' + el.children.length + ' total)');
return kids;
}
function getRelevantStyles(el) {
var cs = window.getComputedStyle(el);
var pick = ['display','position','width','height','color','background-color','font-size','font-family','padding','margin','border','flex','grid-template-columns','gap','overflow','opacity','z-index'];
var out = {};
pick.forEach(function(p) {
var v = cs.getPropertyValue(p);
if (v && v !== 'none' && v !== 'normal' && v !== 'auto' && v !== '0px' && v !== 'rgba(0, 0, 0, 0)') out[p] = v;
});
return out;
}
function onClick(e) {
e.preventDefault();
e.stopPropagation();
e.stopImmediatePropagation();
var el = e.target;
if (last) last.classList.remove('__paseo-hover');
hoverLabel.style.display = 'none';
var attrs = {};
for (var i = 0; i < el.attributes.length; i++) {
attrs[el.attributes[i].name] = el.attributes[i].value;
}
var rect = el.getBoundingClientRect();
var result = {
tag: el.tagName.toLowerCase(),
text: (el.innerText || '').substring(0, 500),
selector: buildSelector(el),
attributes: attrs,
url: location.href,
outerHTML: el.outerHTML.substring(0, 2000),
computedStyles: getRelevantStyles(el),
__paseoSessionToken: sessionToken,
boundingRect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
reactSource: getReactSource(el),
parentChain: getParentChain(el, 5),
children: getChildSummary(el, 8)
};
destroy();
window.__paseoSelectorResult = result;
}
function onKey(e) {
if (e.key === 'Escape') {
destroy();
window.__paseoSelectorResult = { __cancelled: true, __paseoSessionToken: sessionToken };
}
}
function blockEvent(e) {
e.preventDefault();
e.stopPropagation();
e.stopImmediatePropagation();
}
function destroy() {
document.removeEventListener('mousemove', onMove, true);
document.removeEventListener('click', onClick, true);
document.removeEventListener('keydown', onKey, true);
document.removeEventListener('mousedown', blockEvent, true);
document.removeEventListener('mouseup', blockEvent, true);
document.removeEventListener('pointerdown', blockEvent, true);
document.removeEventListener('pointerup', blockEvent, true);
document.removeEventListener('touchstart', blockEvent, true);
document.removeEventListener('touchend', blockEvent, true);
document.removeEventListener('focus', blockEvent, true);
document.removeEventListener('submit', blockEvent, true);
document.documentElement.classList.remove('__paseo-select-mode');
if (last) last.classList.remove('__paseo-hover');
if (hoverLabel.parentNode) hoverLabel.parentNode.removeChild(hoverLabel);
style.remove();
window.__paseoSelector = null;
}
document.addEventListener('mousemove', onMove, true);
document.addEventListener('click', onClick, true);
document.addEventListener('keydown', onKey, true);
document.addEventListener('mousedown', blockEvent, true);
document.addEventListener('mouseup', blockEvent, true);
document.addEventListener('pointerdown', blockEvent, true);
document.addEventListener('pointerup', blockEvent, true);
document.addEventListener('touchstart', blockEvent, true);
document.addEventListener('touchend', blockEvent, true);
document.addEventListener('focus', blockEvent, true);
document.addEventListener('submit', blockEvent, true);
window.__paseoSelector = { destroy: destroy, sessionToken: sessionToken };
return { installed: true, sessionToken: sessionToken };
})()
`;
}
export function createElementSelectorController(): ElementSelectorController {
let currentSession: ElementSelectorSession | null = null;
let sessionCounter = 0;
const finish = (
session: ElementSelectorSession,
outcome: ElementSelectorOutcome,
guestCleanup: "clear" | "destroy" | null,
): boolean => {
stopSessionTimers(session);
if (currentSession !== session) {
return false;
}
currentSession = null;
if (guestCleanup === "clear") {
clearWebviewSelector(session.webview, session.token);
} else if (guestCleanup === "destroy") {
destroyWebviewSelector(session.webview, session.token);
}
session.onFinish(outcome);
return true;
};
return {
start({ webview, mode, onFinish }) {
if (!webview.isConnected) {
return "unavailable";
}
if (webview.isLoading?.()) {
return "loading";
}
if (currentSession) {
finish(currentSession, { type: "cancelled" }, "clear");
}
sessionCounter += 1;
const session: ElementSelectorSession = {
mode,
token: `${sessionCounter}:${crypto.randomUUID()}`,
webview,
onFinish,
};
currentSession = session;
session.timeoutId = window.setTimeout(() => {
finish(session, { type: "failed", reason: "timeout" }, "destroy");
}, SELECTOR_TIMEOUT_MS);
void executeWebviewJavaScript(webview, buildElementSelectorScript(session.token))
.then((installation) => {
const installationState = readSelectorInstallation(installation, session.token);
if (currentSession !== session) {
if (installationState === "installed") {
destroyWebviewSelector(webview, session.token);
}
return undefined;
}
if (installationState !== "installed") {
finish(session, { type: "failed", reason: installationState }, null);
return undefined;
}
session.stopPolling = startSelectorResultPolling({
webview,
sessionToken: session.token,
onResult: (selection) => {
finish(
session,
selection
? { type: "selected", mode: session.mode, selection }
: { type: "cancelled" },
null,
);
},
});
return undefined;
})
.catch(() => {
finish(session, { type: "failed", reason: "unavailable" }, "destroy");
return undefined;
});
return "started";
},
cancel() {
if (currentSession) {
finish(currentSession, { type: "cancelled" }, "clear");
}
},
stopForWebview(webview) {
if (currentSession?.webview === webview) {
finish(currentSession, { type: "cancelled" }, "destroy");
}
},
};
}
@@ -59,7 +59,6 @@ import {
} from "@/desktop/browser/store"; } from "@/desktop/browser/store";
import { import {
applyInactiveBrowserWebviewViewport, applyInactiveBrowserWebviewViewport,
isResidentBrowserWebviewReady,
prepareBrowserWebview, prepareBrowserWebview,
presentBrowserWebview, presentBrowserWebview,
rememberBrowserWebviewSize, rememberBrowserWebviewSize,
@@ -67,6 +66,11 @@ import {
removeResidentBrowserWebview, removeResidentBrowserWebview,
takeResidentBrowserWebview, takeResidentBrowserWebview,
} from "../resident-webviews"; } from "../resident-webviews";
import {
createElementSelectorController,
type BrowserElementSelection,
type ElementSelectorOutcome,
} from "./element-selector.electron";
type ElectronWebview = HTMLElement & { type ElectronWebview = HTMLElement & {
canGoBack?: () => boolean; canGoBack?: () => boolean;
@@ -77,6 +81,7 @@ type ElectronWebview = HTMLElement & {
stop?: () => void; stop?: () => void;
loadURL?: (url: string) => Promise<void>; loadURL?: (url: string) => Promise<void>;
getURL?: () => string; getURL?: () => string;
isLoading?: () => boolean;
executeJavaScript?: (code: string) => Promise<unknown>; executeJavaScript?: (code: string) => Promise<unknown>;
focus?: () => void; focus?: () => void;
addEventListener: (type: string, listener: EventListenerOrEventListenerObject) => void; addEventListener: (type: string, listener: EventListenerOrEventListenerObject) => void;
@@ -87,10 +92,6 @@ type WebTextInput = TextInput & {
getNativeRef?: () => unknown; getNativeRef?: () => unknown;
}; };
type BrowserElementSelection = Omit<BrowserElementAttachment, "formatted" | "comment"> & {
attributes?: Record<string, string>;
};
interface BrowserElementAnnotation { interface BrowserElementAnnotation {
comment: string; comment: string;
} }
@@ -315,20 +316,6 @@ function executeWebviewJavaScript(webview: ElectronWebview, code: string): Promi
function ignoreWebviewJavaScriptError() {} function ignoreWebviewJavaScriptError() {}
function destroyWebviewSelector(webview: ElectronWebview): void {
void executeWebviewJavaScript(
webview,
"if(window.__paseoSelector) window.__paseoSelector.destroy();",
).catch(ignoreWebviewJavaScriptError);
}
function clearWebviewSelector(webview: ElectronWebview): void {
void executeWebviewJavaScript(
webview,
"if(window.__paseoSelector) window.__paseoSelector.destroy(); window.__paseoSelectorResult = null;",
).catch(ignoreWebviewJavaScriptError);
}
interface BrowserAnnotationMarker { interface BrowserAnnotationMarker {
index: number; index: number;
selector: string; selector: string;
@@ -437,38 +424,6 @@ function isDesktopBrowserShortcutEvent(payload: unknown): payload is DesktopBrow
return event.action === "focus-url"; return event.action === "focus-url";
} }
function startSelectorResultPolling(input: {
webview: ElectronWebview;
onSelection: (selection: BrowserElementSelection) => void;
onDone: () => void;
}): number {
const { webview, onSelection, onDone } = input;
const poll = window.setInterval(() => {
void (async () => {
try {
const raw = await executeWebviewJavaScript(
webview,
"JSON.stringify(window.__paseoSelectorResult || null)",
);
const result = typeof raw === "string" ? JSON.parse(raw) : null;
if (!result) {
return;
}
window.clearInterval(poll);
onDone();
await executeWebviewJavaScript(webview, "window.__paseoSelectorResult = null;");
if (!result.__cancelled) {
onSelection(result as BrowserElementSelection);
}
} catch {
// Keep polling; cross-origin/webview timing can make this transient.
}
})();
}, 200);
return poll;
}
function ToolbarButton({ function ToolbarButton({
label, label,
children, children,
@@ -655,13 +610,15 @@ export function BrowserPane({
const browserRef = useRef(browser); const browserRef = useRef(browser);
browserRef.current = browser; browserRef.current = browser;
const pendingNavigationUrlRef = useRef<string | null>(null); const pendingNavigationUrlRef = useRef<string | null>(null);
const domReadyRef = useRef(false);
const annotationMarkersRef = useRef<BrowserAnnotationMarker[]>([]); const annotationMarkersRef = useRef<BrowserAnnotationMarker[]>([]);
const [selectorMode, setSelectorMode] = useState<"annotate" | "screenshot" | null>(null); const [selectorMode, setSelectorMode] = useState<"annotate" | "screenshot" | null>(null);
const selectorControllerRef = useRef<ReturnType<typeof createElementSelectorController> | null>(
null,
);
if (!selectorControllerRef.current) {
selectorControllerRef.current = createElementSelectorController();
}
const selectorActive = selectorMode !== null; const selectorActive = selectorMode !== null;
// Which action the active selector performs on click: open the annotation card
// ("annotate") or copy a screenshot of the element to the clipboard ("screenshot").
const selectorModeRef = useRef<"annotate" | "screenshot">("annotate");
const toast = useToast(); const toast = useToast();
const toastRef = useRef(toast); const toastRef = useRef(toast);
toastRef.current = toast; toastRef.current = toast;
@@ -737,7 +694,7 @@ export function BrowserPane({
const syncNavigationState = useCallback((input?: { syncUrl?: boolean }) => { const syncNavigationState = useCallback((input?: { syncUrl?: boolean }) => {
const webview = webviewRef.current; const webview = webviewRef.current;
if (!webview || !domReadyRef.current) { if (!webview) {
return; return;
} }
@@ -776,7 +733,6 @@ export function BrowserPane({
const residentWebview = takeResidentBrowserWebview(browserId) as ElectronWebview | null; const residentWebview = takeResidentBrowserWebview(browserId) as ElectronWebview | null;
const webview = residentWebview ?? (document.createElement("webview") as ElectronWebview); const webview = residentWebview ?? (document.createElement("webview") as ElectronWebview);
webviewRef.current = webview; webviewRef.current = webview;
domReadyRef.current = isResidentBrowserWebviewReady(webview);
if (!residentWebview) { if (!residentWebview) {
prepareBrowserWebview(webview, { prepareBrowserWebview(webview, {
browserId, browserId,
@@ -808,7 +764,7 @@ export function BrowserPane({
}); });
const handleStartLoading = () => { const handleStartLoading = () => {
domReadyRef.current = false; selectorControllerRef.current?.stopForWebview(webview);
updateBrowser(browserId, { isLoading: true, lastError: null }); updateBrowser(browserId, { isLoading: true, lastError: null });
syncNavigationState({ syncUrl: false }); syncNavigationState({ syncUrl: false });
}; };
@@ -875,7 +831,6 @@ export function BrowserPane({
}); });
}; };
const handleDomReady = () => { const handleDomReady = () => {
domReadyRef.current = true;
syncNavigationState(); syncNavigationState();
// The previous page's overlay is gone after a load; re-apply markers for // The previous page's overlay is gone after a load; re-apply markers for
// the freshly loaded document. // the freshly loaded document.
@@ -940,10 +895,10 @@ export function BrowserPane({
} else { } else {
removeResidentBrowserWebview(browserIdRef.current); removeResidentBrowserWebview(browserIdRef.current);
} }
selectorControllerRef.current?.stopForWebview(webview);
if (webviewRef.current === webview) { if (webviewRef.current === webview) {
webviewRef.current = null; webviewRef.current = null;
} }
domReadyRef.current = false;
}; };
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
}, [browserId, onFocusPane]); }, [browserId, onFocusPane]);
@@ -1196,8 +1151,8 @@ export function BrowserPane({
); );
const handleSelectorResult = useCallback( const handleSelectorResult = useCallback(
(selection: BrowserElementSelection) => { (selection: BrowserElementSelection, mode: "annotate" | "screenshot") => {
if (selectorModeRef.current === "screenshot") { if (mode === "screenshot") {
void screenshotElementToClipboard(selection); void screenshotElementToClipboard(selection);
return; return;
} }
@@ -1230,271 +1185,62 @@ export function BrowserPane({
setPendingSelection(null); setPendingSelection(null);
}, []); }, []);
const showSelectorFailure = useCallback(
(reason: "loading" | "timeout" | "unavailable") => {
const message =
reason === "loading"
? t("workspace.browser.controls.selectorLoading")
: t("workspace.browser.controls.selectorFailed");
toastRef.current?.error(message);
},
[t],
);
const handleElementSelectorOutcome = useCallback(
(outcome: ElementSelectorOutcome) => {
setSelectorMode(null);
if (outcome.type === "selected") {
handleSelectorResult(outcome.selection, outcome.mode);
} else if (outcome.type === "failed") {
showSelectorFailure(outcome.reason);
}
},
[handleSelectorResult, showSelectorFailure],
);
const startElementSelector = useCallback( const startElementSelector = useCallback(
(mode: "annotate" | "screenshot") => { (mode: "annotate" | "screenshot") => {
if (mode === "annotate" && !workspaceAttachmentScopeKey) {
showSelectorFailure("unavailable");
return;
}
const webview = webviewRef.current; const webview = webviewRef.current;
if (!webview || !domReadyRef.current) return; const controller = selectorControllerRef.current;
// Annotate needs a workspace scope to attach to; screenshot only copies. if (!webview || !controller) {
if (mode === "annotate" && !workspaceAttachmentScopeKey) return; showSelectorFailure("unavailable");
selectorModeRef.current = mode; return;
}
const startResult = controller.start({
webview,
mode,
onFinish: handleElementSelectorOutcome,
});
if (startResult !== "started") {
showSelectorFailure(startResult);
return;
}
pendingScreenshotRef.current = undefined; pendingScreenshotRef.current = undefined;
setPendingSelection(null); setPendingSelection(null);
setSelectorMode(mode); setSelectorMode(mode);
const js = `
(function() {
if (window.__paseoSelector) { window.__paseoSelector.destroy(); }
var overlay = null;
var style = document.createElement('style');
style.textContent = [
'.__paseo-hover { outline: 2px solid #3b82f6 !important; outline-offset: 2px !important; cursor: crosshair !important; }',
'.__paseo-select-mode, .__paseo-select-mode * { cursor: crosshair !important; pointer-events: auto !important; user-select: none !important; }',
'.__paseo-select-mode *, .__paseo-select-mode *::before, .__paseo-select-mode *::after { animation: none !important; transition: none !important; }',
'.__paseo-select-mode a, .__paseo-select-mode button, .__paseo-select-mode input, .__paseo-select-mode select, .__paseo-select-mode textarea, .__paseo-select-mode [role="button"], .__paseo-select-mode [onclick] { pointer-events: none !important; }',
'.__paseo-select-mode iframe, .__paseo-select-mode video, .__paseo-select-mode audio { pointer-events: none !important; }',
'.__paseo-hover-label { position: fixed; z-index: 2147483647; pointer-events: none; max-width: 360px; padding: 4px 8px; border-radius: 6px; background: rgba(24,24,27,0.96); color: #fff; font: 500 11px/1.45 ui-monospace,SFMono-Regular,Menlo,monospace; box-shadow: 0 2px 10px rgba(0,0,0,0.35); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }',
'.__paseo-hover-label .__paseo-tag { color: #93c5fd; }',
'.__paseo-hover-label .__paseo-id { color: #fca5a5; }',
'.__paseo-hover-label .__paseo-cls { color: #fcd34d; }',
'.__paseo-hover-label .__paseo-dim { color: #a1a1aa; margin-left: 6px; }',
'.__paseo-hover-label .__paseo-comp { color: #86efac; margin-left: 6px; }',
].join('\\n');
document.head.appendChild(style);
document.documentElement.classList.add('__paseo-select-mode');
var hoverLabel = document.createElement('div');
hoverLabel.className = '__paseo-hover-label';
hoverLabel.style.display = 'none';
document.documentElement.appendChild(hoverLabel);
var last = null;
function escapeHtml(value) {
return String(value).replace(/[&<>"]/g, function(ch) {
return ch === '&' ? '&amp;' : ch === '<' ? '&lt;' : ch === '>' ? '&gt;' : '&quot;';
});
}
function describeElement(el) {
var tag = el.tagName ? el.tagName.toLowerCase() : 'node';
var parts = ['<span class="__paseo-tag">' + escapeHtml(tag) + '</span>'];
if (el.id) {
parts.push('<span class="__paseo-id">#' + escapeHtml(el.id) + '</span>');
}
if (el.classList && el.classList.length) {
var cls = Array.prototype.slice.call(el.classList, 0, 2)
.filter(function(c) { return c.indexOf('__paseo') !== 0; })
.map(function(c) { return '.' + escapeHtml(c); })
.join('');
if (cls) parts.push('<span class="__paseo-cls">' + cls + '</span>');
}
var comp = getReactSource(el);
if (comp && comp.componentName) {
parts.push('<span class="__paseo-comp">&lt;' + escapeHtml(comp.componentName) + '&gt;</span>');
}
var rect = el.getBoundingClientRect();
parts.push('<span class="__paseo-dim">' + Math.round(rect.width) + '×' + Math.round(rect.height) + '</span>');
return { html: parts.join(''), rect: rect };
}
function positionLabel(rect, e) {
var pad = 12;
var lw = hoverLabel.offsetWidth || 0;
var lh = hoverLabel.offsetHeight || 0;
var top = rect.top - lh - 6;
if (top < 4) top = rect.bottom + 6;
if (top + lh > window.innerHeight - 4) top = Math.max(4, e.clientY - lh - 6);
var left = rect.left;
if (left + lw > window.innerWidth - 4) left = Math.max(4, window.innerWidth - lw - 4);
if (left < 4) left = 4;
hoverLabel.style.top = Math.round(top) + 'px';
hoverLabel.style.left = Math.round(left) + 'px';
}
function onMove(e) {
e.preventDefault();
e.stopPropagation();
if (last) last.classList.remove('__paseo-hover');
var el = e.target;
el.classList.add('__paseo-hover');
last = el;
try {
var info = describeElement(el);
hoverLabel.innerHTML = info.html;
hoverLabel.style.display = 'block';
positionLabel(info.rect, e);
} catch (err) {
hoverLabel.style.display = 'none';
}
}
function buildSelector(el) {
if (el.id) return '#' + el.id;
var path = [];
while (el && el.nodeType === 1) {
var seg = el.tagName.toLowerCase();
if (el.id) { path.unshift('#' + el.id); break; }
var sib = el, nth = 1;
while (sib = sib.previousElementSibling) { if (sib.tagName === el.tagName) nth++; }
if (nth > 1) seg += ':nth-of-type(' + nth + ')';
path.unshift(seg);
el = el.parentElement;
}
return path.join(' > ');
}
function getReactSource(el) {
var keys = Object.keys(el);
for (var i = 0; i < keys.length; i++) {
if (keys[i].startsWith('__reactFiber$') || keys[i].startsWith('__reactInternalInstance$')) {
var fiber = el[keys[i]];
while (fiber) {
if (fiber._debugSource) {
return {
fileName: fiber._debugSource.fileName || null,
lineNumber: fiber._debugSource.lineNumber || null,
columnNumber: fiber._debugSource.columnNumber || null,
componentName: (fiber.type && (typeof fiber.type === 'string' ? fiber.type : fiber.type.displayName || fiber.type.name)) || null
};
}
if (fiber._debugOwner) { fiber = fiber._debugOwner; }
else if (fiber.return) { fiber = fiber.return; }
else break;
}
}
}
return null;
}
function getParentChain(el, depth) {
var chain = [];
var cur = el.parentElement;
for (var i = 0; i < (depth || 5) && cur; i++) {
var desc = cur.tagName.toLowerCase();
if (cur.id) desc += '#' + cur.id;
if (cur.className && typeof cur.className === 'string') { var cls = cur.className.trim().replace(/ +/g, ' ').split(' ').slice(0,2).join('.'); if (cls) desc += '.' + cls; }
chain.push(desc);
cur = cur.parentElement;
}
return chain;
}
function getChildSummary(el, max) {
var kids = [];
for (var i = 0; i < Math.min(el.children.length, max || 8); i++) {
var c = el.children[i];
var desc = c.tagName.toLowerCase();
if (c.id) desc += '#' + c.id;
kids.push(desc);
}
if (el.children.length > (max || 8)) kids.push('...(' + el.children.length + ' total)');
return kids;
}
function getRelevantStyles(el) {
var cs = window.getComputedStyle(el);
var pick = ['display','position','width','height','color','background-color','font-size','font-family','padding','margin','border','flex','grid-template-columns','gap','overflow','opacity','z-index'];
var out = {};
pick.forEach(function(p) {
var v = cs.getPropertyValue(p);
if (v && v !== 'none' && v !== 'normal' && v !== 'auto' && v !== '0px' && v !== 'rgba(0, 0, 0, 0)') out[p] = v;
});
return out;
}
function onClick(e) {
e.preventDefault();
e.stopPropagation();
e.stopImmediatePropagation();
var el = e.target;
if (last) last.classList.remove('__paseo-hover');
hoverLabel.style.display = 'none';
var attrs = {};
for (var i = 0; i < el.attributes.length; i++) {
attrs[el.attributes[i].name] = el.attributes[i].value;
}
var rect = el.getBoundingClientRect();
var result = {
tag: el.tagName.toLowerCase(),
text: (el.innerText || '').substring(0, 500),
selector: buildSelector(el),
attributes: attrs,
url: location.href,
outerHTML: el.outerHTML.substring(0, 2000),
computedStyles: getRelevantStyles(el),
boundingRect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
reactSource: getReactSource(el),
parentChain: getParentChain(el, 5),
children: getChildSummary(el, 8)
};
destroy();
window.__paseoSelectorResult = result;
}
function onKey(e) {
if (e.key === 'Escape') { destroy(); window.__paseoSelectorResult = { __cancelled: true }; }
}
function blockEvent(e) {
e.preventDefault();
e.stopPropagation();
e.stopImmediatePropagation();
}
function destroy() {
document.removeEventListener('mousemove', onMove, true);
document.removeEventListener('click', onClick, true);
document.removeEventListener('keydown', onKey, true);
document.removeEventListener('mousedown', blockEvent, true);
document.removeEventListener('mouseup', blockEvent, true);
document.removeEventListener('pointerdown', blockEvent, true);
document.removeEventListener('pointerup', blockEvent, true);
document.removeEventListener('touchstart', blockEvent, true);
document.removeEventListener('touchend', blockEvent, true);
document.removeEventListener('focus', blockEvent, true);
document.removeEventListener('submit', blockEvent, true);
document.documentElement.classList.remove('__paseo-select-mode');
if (last) last.classList.remove('__paseo-hover');
if (hoverLabel.parentNode) hoverLabel.parentNode.removeChild(hoverLabel);
style.remove();
window.__paseoSelector = null;
}
document.addEventListener('mousemove', onMove, true);
document.addEventListener('click', onClick, true);
document.addEventListener('keydown', onKey, true);
document.addEventListener('mousedown', blockEvent, true);
document.addEventListener('mouseup', blockEvent, true);
document.addEventListener('pointerdown', blockEvent, true);
document.addEventListener('pointerup', blockEvent, true);
document.addEventListener('touchstart', blockEvent, true);
document.addEventListener('touchend', blockEvent, true);
document.addEventListener('focus', blockEvent, true);
document.addEventListener('submit', blockEvent, true);
window.__paseoSelector = { destroy: destroy };
})()
`;
try {
void executeWebviewJavaScript(webview, js)
.then(() => {
const poll = startSelectorResultPolling({
webview,
onSelection: handleSelectorResult,
onDone: () => setSelectorMode(null),
});
window.setTimeout(() => {
window.clearInterval(poll);
setSelectorMode(null);
if (webviewRef.current !== webview || !domReadyRef.current) {
return;
}
destroyWebviewSelector(webview);
}, 30000);
return undefined;
})
.catch(() => {
setSelectorMode(null);
});
} catch {
setSelectorMode(null);
}
}, },
[handleSelectorResult, workspaceAttachmentScopeKey], [handleElementSelectorOutcome, showSelectorFailure, workspaceAttachmentScopeKey],
); );
const cancelElementSelector = useCallback(() => { const cancelElementSelector = useCallback(() => {
const webview = webviewRef.current; selectorControllerRef.current?.cancel();
setSelectorMode(null); setSelectorMode(null);
if (webview && domReadyRef.current) {
try {
clearWebviewSelector(webview);
} catch {}
}
}, []); }, []);
const currentPageUrl = browser?.url ?? null; const currentPageUrl = browser?.url ?? null;
@@ -1526,7 +1272,7 @@ export function BrowserPane({
return; return;
} }
const webview = webviewRef.current; const webview = webviewRef.current;
if (!webview || !domReadyRef.current) { if (!webview) {
return; return;
} }
if (annotationMarkers.length === 0) { if (annotationMarkers.length === 0) {
@@ -5,7 +5,6 @@ import {
clearResidentBrowserWebviewsForTests, clearResidentBrowserWebviewsForTests,
ensureResidentBrowserWebview, ensureResidentBrowserWebview,
getResidentBrowserWebview, getResidentBrowserWebview,
isResidentBrowserWebviewReady,
prepareBrowserWebview, prepareBrowserWebview,
presentBrowserWebview, presentBrowserWebview,
rememberBrowserWebviewSize, rememberBrowserWebviewSize,
@@ -205,28 +204,6 @@ describe("resident browser webviews", () => {
expect(webview.parentElement.style.height).toBe("600px"); expect(webview.parentElement.style.height).toBe("600px");
}); });
it("retains document readiness with the resident webview", () => {
const webview = ensureTestBrowser({
browserId: "browser-ready",
workspaceId: "workspace-ready",
url: "https://example.com",
});
if (!webview) {
throw new Error("Expected resident browser webview");
}
expect(isResidentBrowserWebviewReady(webview)).toBe(false);
webview.dispatchEvent(new Event("dom-ready"));
releaseResidentBrowserWebview("browser-ready", webview);
const reusedWebview = takeResidentBrowserWebview("browser-ready");
expect(reusedWebview).toBe(webview);
expect(isResidentBrowserWebviewReady(reusedWebview as HTMLElement)).toBe(true);
webview.dispatchEvent(new Event("did-start-loading"));
expect(isResidentBrowserWebviewReady(webview)).toBe(false);
});
it("creates a resident webview for an agent-created unfocused tab", () => { it("creates a resident webview for an agent-created unfocused tab", () => {
const webview = ensureTestBrowser({ const webview = ensureTestBrowser({
browserId: "browser-agent", browserId: "browser-agent",
@@ -15,7 +15,6 @@ const RESIDENT_VIEWPORT_HEIGHT = 800;
const residentWebviewsByBrowserId = new Map<string, HTMLElement>(); const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
const residentSurfacesByBrowserId = new Map<string, HTMLElement>(); const residentSurfacesByBrowserId = new Map<string, HTMLElement>();
const residentWebviewSizesByBrowserId = new Map<string, { width: number; height: number }>(); const residentWebviewSizesByBrowserId = new Map<string, { width: number; height: number }>();
const readyResidentWebviews = new WeakSet<HTMLElement>();
interface BrowserWebviewElement extends HTMLElement { interface BrowserWebviewElement extends HTMLElement {
src: string; src: string;
@@ -75,15 +74,6 @@ function registerBrowserWhenAttached(
}); });
} }
function registerBrowserReadiness(webview: HTMLElement): void {
webview.addEventListener("did-start-loading", () => {
readyResidentWebviews.delete(webview);
});
webview.addEventListener("dom-ready", () => {
readyResidentWebviews.add(webview);
});
}
function trimNonEmpty(value: string | null | undefined): string | null { function trimNonEmpty(value: string | null | undefined): string | null {
if (typeof value !== "string") { if (typeof value !== "string") {
return null; return null;
@@ -322,11 +312,6 @@ export function prepareBrowserWebview(
(webview as BrowserWebviewElement).src = input.initialUrl; (webview as BrowserWebviewElement).src = input.initialUrl;
} }
registerBrowserWhenAttached(webview as BrowserWebviewElement, input, browser); registerBrowserWhenAttached(webview as BrowserWebviewElement, input, browser);
registerBrowserReadiness(webview);
}
export function isResidentBrowserWebviewReady(webview: HTMLElement): boolean {
return readyResidentWebviews.has(webview);
} }
export function ensureResidentBrowserWebview(input: { export function ensureResidentBrowserWebview(input: {
+2
View File
@@ -523,6 +523,8 @@ export const ar: TranslationResources = {
screenshotCopied: "تم نسخ لقطة الشاشة إلى الحافظة", screenshotCopied: "تم نسخ لقطة الشاشة إلى الحافظة",
elementCopied: "تم نسخ العنصر إلى الحافظة", elementCopied: "تم نسخ العنصر إلى الحافظة",
screenshotFailed: "تعذّر نسخ لقطة الشاشة", screenshotFailed: "تعذّر نسخ لقطة الشاشة",
selectorLoading: "انتظر حتى ينتهي تحميل الصفحة",
selectorFailed: "تعذّر بدء محدد العنصر",
}, },
annotate: { annotate: {
title: "التعليق على العنصر", title: "التعليق على العنصر",
+2
View File
@@ -522,6 +522,8 @@ export const en = {
screenshotCopied: "Copied screenshot to clipboard", screenshotCopied: "Copied screenshot to clipboard",
elementCopied: "Copied element to clipboard", elementCopied: "Copied element to clipboard",
screenshotFailed: "Couldn't copy screenshot", screenshotFailed: "Couldn't copy screenshot",
selectorLoading: "Wait for the page to finish loading",
selectorFailed: "Couldn't start element selector",
}, },
annotate: { annotate: {
title: "Annotate element", title: "Annotate element",
+2
View File
@@ -527,6 +527,8 @@ export const es: TranslationResources = {
screenshotCopied: "Captura copiada al portapapeles", screenshotCopied: "Captura copiada al portapapeles",
elementCopied: "Elemento copiado al portapapeles", elementCopied: "Elemento copiado al portapapeles",
screenshotFailed: "No se pudo copiar la captura", screenshotFailed: "No se pudo copiar la captura",
selectorLoading: "Espera a que la página termine de cargar",
selectorFailed: "No se pudo iniciar el selector de elementos",
}, },
annotate: { annotate: {
title: "Anotar elemento", title: "Anotar elemento",
+2
View File
@@ -527,6 +527,8 @@ export const fr: TranslationResources = {
screenshotCopied: "Capture d'écran copiée dans le presse-papiers", screenshotCopied: "Capture d'écran copiée dans le presse-papiers",
elementCopied: "Élément copié dans le presse-papiers", elementCopied: "Élément copié dans le presse-papiers",
screenshotFailed: "Impossible de copier la capture", screenshotFailed: "Impossible de copier la capture",
selectorLoading: "Attendez la fin du chargement de la page",
selectorFailed: "Impossible de démarrer le sélecteur d’élément",
}, },
annotate: { annotate: {
title: "Annoter l'élément", title: "Annoter l'élément",
+2
View File
@@ -527,6 +527,8 @@ export const ja: TranslationResources = {
screenshotCopied: "スクリーンショットをクリップボードにコピーしました", screenshotCopied: "スクリーンショットをクリップボードにコピーしました",
elementCopied: "要素をクリップボードにコピーしました", elementCopied: "要素をクリップボードにコピーしました",
screenshotFailed: "スクリーンショットをコピーできませんでした", screenshotFailed: "スクリーンショットをコピーできませんでした",
selectorLoading: "ページの読み込みが完了するまでお待ちください",
selectorFailed: "要素セレクターを開始できませんでした",
}, },
annotate: { annotate: {
title: "要素に注釈を付ける", title: "要素に注釈を付ける",
+2
View File
@@ -524,6 +524,8 @@ export const ko: TranslationResources = {
screenshotCopied: "스크린샷을 클립보드에 복사했습니다.", screenshotCopied: "스크린샷을 클립보드에 복사했습니다.",
elementCopied: "요소를 클립보드에 복사했습니다.", elementCopied: "요소를 클립보드에 복사했습니다.",
screenshotFailed: "스크린샷을 복사할 수 없습니다.", screenshotFailed: "스크린샷을 복사할 수 없습니다.",
selectorLoading: "페이지 로딩이 끝날 때까지 기다려 주세요.",
selectorFailed: "요소 선택기를 시작할 수 없습니다.",
}, },
annotate: { annotate: {
title: "요소에 주석 달기", title: "요소에 주석 달기",
+2
View File
@@ -527,6 +527,8 @@ export const ptBR: TranslationResources = {
screenshotCopied: "Captura copiada para a área de transferência", screenshotCopied: "Captura copiada para a área de transferência",
elementCopied: "Elemento copiado para a área de transferência", elementCopied: "Elemento copiado para a área de transferência",
screenshotFailed: "Não foi possível copiar a captura", screenshotFailed: "Não foi possível copiar a captura",
selectorLoading: "Aguarde o carregamento da página terminar",
selectorFailed: "Não foi possível iniciar o seletor de elemento",
}, },
annotate: { annotate: {
title: "Anotar elemento", title: "Anotar elemento",
+2
View File
@@ -527,6 +527,8 @@ export const ru: TranslationResources = {
screenshotCopied: "Снимок скопирован в буфер обмена", screenshotCopied: "Снимок скопирован в буфер обмена",
elementCopied: "Элемент скопирован в буфер обмена", elementCopied: "Элемент скопирован в буфер обмена",
screenshotFailed: "Не удалось скопировать снимок", screenshotFailed: "Не удалось скопировать снимок",
selectorLoading: "Дождитесь окончания загрузки страницы",
selectorFailed: "Не удалось запустить выбор элемента",
}, },
annotate: { annotate: {
title: "Аннотировать элемент", title: "Аннотировать элемент",
+2
View File
@@ -523,6 +523,8 @@ export const zhCN: TranslationResources = {
screenshotCopied: "已将截图复制到剪贴板", screenshotCopied: "已将截图复制到剪贴板",
elementCopied: "已将元素复制到剪贴板", elementCopied: "已将元素复制到剪贴板",
screenshotFailed: "无法复制截图", screenshotFailed: "无法复制截图",
selectorLoading: "请等待页面加载完成",
selectorFailed: "无法启动元素选择器",
}, },
annotate: { annotate: {
title: "标注元素", title: "标注元素",
+98 -7
View File
@@ -232,6 +232,21 @@ async function callBrowserTool(client, name, args = {}) {
return mcpPayload(await client.callTool({ name, args }), name); return mcpPayload(await client.callTool({ name, args }), name);
} }
async function waitForGuestSelector(client, browserId) {
const deadline = Date.now() + 5_000;
while (Date.now() < deadline) {
const evaluated = await callBrowserTool(client, "browser_evaluate", {
browserId,
function: "() => Boolean(globalThis.__paseoSelector)",
});
if (JSON.parse(evaluated.resultJson) === true) {
return true;
}
await delay(50);
}
return false;
}
async function createCallerAgent(daemonPort) { async function createCallerAgent(daemonPort) {
const transport = new StreamableHTTPClientTransport( const transport = new StreamableHTTPClientTransport(
new URL(`http://127.0.0.1:${daemonPort}/mcp/agents`), new URL(`http://127.0.0.1:${daemonPort}/mcp/agents`),
@@ -384,7 +399,7 @@ function recordViewportMismatch(failures, label, actual, expected) {
); );
} }
async function runRegression({ page, client, serverId, targetUrl, callerAgentId }) { async function runRegression({ page, client, serverId, targetUrl, callerAgentId, artifactDir }) {
const failures = []; const failures = [];
const originalWorkspaceId = workspaceIds[0]; const originalWorkspaceId = workspaceIds[0];
const originalWorkspaceRow = page.getByTestId( const originalWorkspaceRow = page.getByTestId(
@@ -673,16 +688,90 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId
responsiveViewport, responsiveViewport,
); );
await originalDeck.getByRole("button", { name: "Annotate element" }).click(); const annotateButton = originalDeck.getByRole("button", { name: "Annotate element" });
await page.waitForTimeout(250); await page.evaluate((id) => {
const selectorResult = await callBrowserTool(client, "browser_evaluate", { const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
if (!(webview instanceof HTMLElement)) throw new Error(`Browser webview ${id} was unavailable`);
globalThis.__paseoOriginalIsLoadingDescriptor = Object.getOwnPropertyDescriptor(
webview,
"isLoading",
);
Object.defineProperty(webview, "isLoading", {
configurable: true,
value: () => true,
});
}, browserId);
await annotateButton.click();
await page.waitForTimeout(100);
assert(
!(await originalDeck.getByRole("button", { name: "Cancel element selector" }).isVisible()),
"Element selector started while the guest reported a genuine load",
);
const selectorDuringLoad = await callBrowserTool(client, "browser_evaluate", {
browserId, browserId,
function: "() => Boolean(globalThis.__paseoSelector)", function: "() => Boolean(globalThis.__paseoSelector)",
}); });
if (JSON.parse(selectorResult.resultJson) !== true) { assert(
failures.push("reused loaded browser remains ready for element annotation after remount"); JSON.parse(selectorDuringLoad.resultJson) === false,
"Selector injected while the guest reported a genuine load",
);
assert(
await page.getByText("Wait for the page to finish loading").isVisible(),
"Element selector loading failure was not visible",
);
await page.evaluate((id) => {
const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
if (!(webview instanceof HTMLElement)) throw new Error(`Browser webview ${id} was unavailable`);
const descriptor = globalThis.__paseoOriginalIsLoadingDescriptor;
if (descriptor) Object.defineProperty(webview, "isLoading", descriptor);
else delete webview.isLoading;
delete globalThis.__paseoOriginalIsLoadingDescriptor;
}, browserId);
const readyStateResult = await callBrowserTool(client, "browser_evaluate", {
browserId,
function: "() => document.readyState",
});
assert(
JSON.parse(readyStateResult.resultJson) === "complete",
"Local browser document did not finish loading",
);
// Reproduce the report's mismatch: the guest is complete, but the pane's
// last loading signal says it is not ready.
await page.evaluate((id) => {
const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`);
if (!(webview instanceof HTMLElement)) {
throw new Error(`Browser webview ${id} was unavailable`);
}
webview.dispatchEvent(new Event("did-start-loading"));
}, browserId);
await annotateButton.click();
const annotateSelectorActive = await waitForGuestSelector(client, browserId);
await page.screenshot({ path: path.join(artifactDir, "local-page-annotate-selector.png") });
if (!annotateSelectorActive) {
failures.push("loaded local browser remains ready for element annotation");
} }
await originalDeck.getByRole("button", { name: "Cancel element selector" }).click();
await delay(10_000);
const screenshotButton = originalDeck.getByRole("button", { name: "Screenshot element" });
await screenshotButton.click();
const screenshotSelectorActive = await waitForGuestSelector(client, browserId);
if (!screenshotSelectorActive) {
failures.push("loaded local browser remains ready for element screenshots");
}
await delay(20_500);
const selectorAfterPriorTimeout = await callBrowserTool(client, "browser_evaluate", {
browserId,
function: "() => Boolean(globalThis.__paseoSelector)",
});
if (JSON.parse(selectorAfterPriorTimeout.resultJson) !== true) {
failures.push("a previous selector timeout does not destroy the current selector session");
}
await page.screenshot({ path: path.join(artifactDir, "local-page-screenshot-selector.png") });
await originalDeck.getByRole("button", { name: "Cancel element selector" }).click();
if (failures.length > 0) { if (failures.length > 0) {
throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`); throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`);
} }
@@ -698,6 +787,7 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId
list: "passed", list: "passed",
snapshot: "passed", snapshot: "passed",
click: "passed", click: "passed",
localPageSelectors: "passed",
}; };
} }
@@ -796,10 +886,11 @@ async function main() {
serverId: status.serverId, serverId: status.serverId,
targetUrl: target.url, targetUrl: target.url,
callerAgentId, callerAgentId,
artifactDir,
}); });
writeJson(path.join(artifactDir, "result.json"), report); writeJson(path.join(artifactDir, "result.json"), report);
console.log( console.log(
`Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click passed.`, `Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click, local-page selectors passed.`,
); );
} catch (error) { } catch (error) {
console.error(`Browser desktop browser E2E failed. Artifacts: ${artifactDir}`); console.error(`Browser desktop browser E2E failed. Artifacts: ${artifactDir}`);