From 770b87a15cab539f658529e39281b26daff9c06d Mon Sep 17 00:00:00 2001 From: "Daniel G. Kang" Date: Fri, 14 Aug 2026 16:51:42 +0900 Subject: [PATCH] fix(desktop): keep element selectors available on loaded pages (#3187) * fix(desktop): keep element selectors available on loaded pages Annotate element and Screenshot element returned early when a cached load signal said the guest was not ready, even if the current document was complete and executable. Let executeJavaScript own the readiness boundary and keep selector cleanup available in the same stale-signal state. Extend the real Electron browser regression to reproduce the complete-document/stale-signal mismatch through both toolbar actions. * test(desktop): await browser selector injection Poll the guest selector state through the browser bridge instead of assuming injection completes within 250 ms. This keeps the Electron regression deterministic on slower CI hosts. * Harden browser selector session lifecycle * Keep selector E2E fixtures independent * fix(desktop): harden selector and annotation lifecycle * fix(desktop): keep selector PR focused * refactor(desktop): own element selector lifecycle --------- Co-authored-by: Daniel G. Kang <180234654+dgk-dev@users.noreply.github.com> Co-authored-by: Mohamed Boudra --- .../browser/pane/element-selector.electron.ts | 470 ++++++++++++++++++ .../desktop/browser/pane/index.electron.tsx | 380 +++----------- .../browser/resident-webviews.browser.test.ts | 23 - .../src/desktop/browser/resident-webviews.ts | 15 - packages/app/src/i18n/resources/ar.ts | 2 + packages/app/src/i18n/resources/en.ts | 2 + packages/app/src/i18n/resources/es.ts | 2 + packages/app/src/i18n/resources/fr.ts | 2 + packages/app/src/i18n/resources/ja.ts | 2 + packages/app/src/i18n/resources/ko.ts | 2 + packages/app/src/i18n/resources/pt-BR.ts | 2 + packages/app/src/i18n/resources/ru.ts | 2 + packages/app/src/i18n/resources/zh-CN.ts | 2 + packages/desktop/e2e/browser-tabs.e2e.mjs | 105 +++- 14 files changed, 649 insertions(+), 362 deletions(-) create mode 100644 packages/app/src/desktop/browser/pane/element-selector.electron.ts diff --git a/packages/app/src/desktop/browser/pane/element-selector.electron.ts b/packages/app/src/desktop/browser/pane/element-selector.electron.ts new file mode 100644 index 000000000..316116828 --- /dev/null +++ b/packages/app/src/desktop/browser/pane/element-selector.electron.ts @@ -0,0 +1,470 @@ +import type { BrowserElementAttachment } from "@/attachments/types"; + +export type BrowserElementSelection = Omit & { + attributes?: Record; +}; + +export type ElementSelectorMode = "annotate" | "screenshot"; + +export type ElementSelectorOutcome = + | { type: "selected"; mode: ElementSelectorMode; selection: BrowserElementSelection } + | { type: "cancelled" } + | { type: "failed"; reason: "loading" | "timeout" | "unavailable" }; + +export type ElementSelectorStartResult = "started" | "loading" | "unavailable"; + +interface ElementSelectorWebview extends HTMLElement { + isLoading?: () => boolean; + executeJavaScript?: (code: string) => Promise; +} + +interface ElementSelectorSession { + mode: ElementSelectorMode; + token: string; + webview: ElementSelectorWebview; + onFinish: (outcome: ElementSelectorOutcome) => void; + stopPolling?: () => void; + timeoutId?: number; +} + +export interface ElementSelectorController { + start(input: { + webview: ElementSelectorWebview; + mode: ElementSelectorMode; + onFinish: (outcome: ElementSelectorOutcome) => void; + }): ElementSelectorStartResult; + cancel(): void; + stopForWebview(webview: ElementSelectorWebview): void; +} + +const SELECTOR_TIMEOUT_MS = 30_000; + +function executeWebviewJavaScript(webview: ElementSelectorWebview, code: string): Promise { + if (!webview.isConnected) { + return Promise.resolve(null); + } + try { + return webview.executeJavaScript?.(code) ?? Promise.resolve(null); + } catch (error) { + return Promise.reject(error); + } +} + +function ignoreWebviewJavaScriptError() {} + +function destroyWebviewSelector(webview: ElementSelectorWebview, sessionToken: string): void { + const token = JSON.stringify(sessionToken); + void executeWebviewJavaScript( + webview, + `if (window.__paseoSelector?.sessionToken === ${token}) window.__paseoSelector.destroy();`, + ).catch(ignoreWebviewJavaScriptError); +} + +function clearWebviewSelector(webview: ElementSelectorWebview, sessionToken: string): void { + const token = JSON.stringify(sessionToken); + void executeWebviewJavaScript( + webview, + `if (window.__paseoSelector?.sessionToken === ${token}) window.__paseoSelector.destroy(); if (window.__paseoSelectorResult?.__paseoSessionToken === ${token}) window.__paseoSelectorResult = null;`, + ).catch(ignoreWebviewJavaScriptError); +} + +function readSelectorInstallation( + value: unknown, + sessionToken: string, +): "installed" | "loading" | "unavailable" { + if (value === null || typeof value !== "object") { + return "unavailable"; + } + if (Reflect.get(value, "sessionToken") !== sessionToken) { + return "unavailable"; + } + if (Reflect.get(value, "installed") === true) { + return "installed"; + } + return Reflect.get(value, "reason") === "document-loading" ? "loading" : "unavailable"; +} + +function stopSessionTimers(session: ElementSelectorSession): void { + session.stopPolling?.(); + session.stopPolling = undefined; + if (session.timeoutId !== undefined) { + window.clearTimeout(session.timeoutId); + session.timeoutId = undefined; + } +} + +function startSelectorResultPolling(input: { + webview: ElementSelectorWebview; + sessionToken: string; + onResult: (selection: BrowserElementSelection | null) => void; +}): () => void { + const { webview, sessionToken, onResult } = input; + const token = JSON.stringify(sessionToken); + let stopped = false; + let timerId: number | undefined; + + const schedule = () => { + if (!stopped) { + timerId = window.setTimeout(poll, 200); + } + }; + const poll = () => { + void (async () => { + try { + const raw = await executeWebviewJavaScript( + webview, + `JSON.stringify(window.__paseoSelectorResult?.__paseoSessionToken === ${token} ? window.__paseoSelectorResult : null)`, + ); + const result = typeof raw === "string" ? JSON.parse(raw) : null; + if (!result) { + schedule(); + return; + } + stopped = true; + await executeWebviewJavaScript( + webview, + `if (window.__paseoSelectorResult?.__paseoSessionToken === ${token}) window.__paseoSelectorResult = null;`, + ).catch(ignoreWebviewJavaScriptError); + const cancelled = result.__cancelled === true; + delete result.__cancelled; + delete result.__paseoSessionToken; + onResult(cancelled ? null : (result as BrowserElementSelection)); + } catch { + schedule(); + } + })(); + }; + + schedule(); + return () => { + stopped = true; + if (timerId !== undefined) { + window.clearTimeout(timerId); + } + }; +} + +function buildElementSelectorScript(sessionToken: string): string { + const token = JSON.stringify(sessionToken); + return ` + (function() { + var sessionToken = ${token}; + if (document.readyState === 'loading' || !document.head || !document.documentElement) { + return { installed: false, reason: 'document-loading', sessionToken: sessionToken }; + } + if (window.__paseoSelector) { window.__paseoSelector.destroy(); } + window.__paseoSelectorResult = null; + var style = document.createElement('style'); + style.textContent = [ + '.__paseo-hover { outline: 2px solid #3b82f6 !important; outline-offset: 2px !important; cursor: crosshair !important; }', + '.__paseo-select-mode, .__paseo-select-mode * { cursor: crosshair !important; pointer-events: auto !important; user-select: none !important; }', + '.__paseo-select-mode *, .__paseo-select-mode *::before, .__paseo-select-mode *::after { animation: none !important; transition: none !important; }', + '.__paseo-select-mode a, .__paseo-select-mode button, .__paseo-select-mode input, .__paseo-select-mode select, .__paseo-select-mode textarea, .__paseo-select-mode [role="button"], .__paseo-select-mode [onclick] { pointer-events: none !important; }', + '.__paseo-select-mode iframe, .__paseo-select-mode video, .__paseo-select-mode audio { pointer-events: none !important; }', + '.__paseo-hover-label { position: fixed; z-index: 2147483647; pointer-events: none; max-width: 360px; padding: 4px 8px; border-radius: 6px; background: rgba(24,24,27,0.96); color: #fff; font: 500 11px/1.45 ui-monospace,SFMono-Regular,Menlo,monospace; box-shadow: 0 2px 10px rgba(0,0,0,0.35); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }', + '.__paseo-hover-label .__paseo-tag { color: #93c5fd; }', + '.__paseo-hover-label .__paseo-id { color: #fca5a5; }', + '.__paseo-hover-label .__paseo-cls { color: #fcd34d; }', + '.__paseo-hover-label .__paseo-dim { color: #a1a1aa; margin-left: 6px; }', + '.__paseo-hover-label .__paseo-comp { color: #86efac; margin-left: 6px; }', + ].join('\\n'); + document.head.appendChild(style); + document.documentElement.classList.add('__paseo-select-mode'); + var hoverLabel = document.createElement('div'); + hoverLabel.className = '__paseo-hover-label'; + hoverLabel.style.display = 'none'; + document.documentElement.appendChild(hoverLabel); + var last = null; + function escapeHtml(value) { + return String(value).replace(/[&<>"]/g, function(ch) { + return ch === '&' ? '&' : ch === '<' ? '<' : ch === '>' ? '>' : '"'; + }); + } + function describeElement(el) { + var tag = el.tagName ? el.tagName.toLowerCase() : 'node'; + var parts = ['' + escapeHtml(tag) + '']; + if (el.id) { + parts.push('#' + escapeHtml(el.id) + ''); + } + if (el.classList && el.classList.length) { + var cls = Array.prototype.slice.call(el.classList, 0, 2) + .filter(function(c) { return c.indexOf('__paseo') !== 0; }) + .map(function(c) { return '.' + escapeHtml(c); }) + .join(''); + if (cls) parts.push('' + cls + ''); + } + var comp = getReactSource(el); + if (comp && comp.componentName) { + parts.push('<' + escapeHtml(comp.componentName) + '>'); + } + var rect = el.getBoundingClientRect(); + parts.push('' + Math.round(rect.width) + '×' + Math.round(rect.height) + ''); + return { html: parts.join(''), rect: rect }; + } + function positionLabel(rect, e) { + var lw = hoverLabel.offsetWidth || 0; + var lh = hoverLabel.offsetHeight || 0; + var top = rect.top - lh - 6; + if (top < 4) top = rect.bottom + 6; + if (top + lh > window.innerHeight - 4) top = Math.max(4, e.clientY - lh - 6); + var left = rect.left; + if (left + lw > window.innerWidth - 4) left = Math.max(4, window.innerWidth - lw - 4); + if (left < 4) left = 4; + hoverLabel.style.top = Math.round(top) + 'px'; + hoverLabel.style.left = Math.round(left) + 'px'; + } + function onMove(e) { + e.preventDefault(); + e.stopPropagation(); + if (last) last.classList.remove('__paseo-hover'); + var el = e.target; + el.classList.add('__paseo-hover'); + last = el; + try { + var info = describeElement(el); + hoverLabel.innerHTML = info.html; + hoverLabel.style.display = 'block'; + positionLabel(info.rect, e); + } catch (err) { + hoverLabel.style.display = 'none'; + } + } + function buildSelector(el) { + if (el.id) return '#' + el.id; + var path = []; + while (el && el.nodeType === 1) { + var seg = el.tagName.toLowerCase(); + if (el.id) { path.unshift('#' + el.id); break; } + var sib = el, nth = 1; + while (sib = sib.previousElementSibling) { if (sib.tagName === el.tagName) nth++; } + if (nth > 1) seg += ':nth-of-type(' + nth + ')'; + path.unshift(seg); + el = el.parentElement; + } + return path.join(' > '); + } + function getReactSource(el) { + var keys = Object.keys(el); + for (var i = 0; i < keys.length; i++) { + if (keys[i].startsWith('__reactFiber$') || keys[i].startsWith('__reactInternalInstance$')) { + var fiber = el[keys[i]]; + while (fiber) { + if (fiber._debugSource) { + return { + fileName: fiber._debugSource.fileName || null, + lineNumber: fiber._debugSource.lineNumber || null, + columnNumber: fiber._debugSource.columnNumber || null, + componentName: (fiber.type && (typeof fiber.type === 'string' ? fiber.type : fiber.type.displayName || fiber.type.name)) || null + }; + } + if (fiber._debugOwner) { fiber = fiber._debugOwner; } + else if (fiber.return) { fiber = fiber.return; } + else break; + } + } + } + return null; + } + function getParentChain(el, depth) { + var chain = []; + var cur = el.parentElement; + for (var i = 0; i < (depth || 5) && cur; i++) { + var desc = cur.tagName.toLowerCase(); + if (cur.id) desc += '#' + cur.id; + if (cur.className && typeof cur.className === 'string') { var cls = cur.className.trim().replace(/ +/g, ' ').split(' ').slice(0,2).join('.'); if (cls) desc += '.' + cls; } + chain.push(desc); + cur = cur.parentElement; + } + return chain; + } + function getChildSummary(el, max) { + var kids = []; + for (var i = 0; i < Math.min(el.children.length, max || 8); i++) { + var c = el.children[i]; + var desc = c.tagName.toLowerCase(); + if (c.id) desc += '#' + c.id; + kids.push(desc); + } + if (el.children.length > (max || 8)) kids.push('...(' + el.children.length + ' total)'); + return kids; + } + function getRelevantStyles(el) { + var cs = window.getComputedStyle(el); + var pick = ['display','position','width','height','color','background-color','font-size','font-family','padding','margin','border','flex','grid-template-columns','gap','overflow','opacity','z-index']; + var out = {}; + pick.forEach(function(p) { + var v = cs.getPropertyValue(p); + if (v && v !== 'none' && v !== 'normal' && v !== 'auto' && v !== '0px' && v !== 'rgba(0, 0, 0, 0)') out[p] = v; + }); + return out; + } + function onClick(e) { + e.preventDefault(); + e.stopPropagation(); + e.stopImmediatePropagation(); + var el = e.target; + if (last) last.classList.remove('__paseo-hover'); + hoverLabel.style.display = 'none'; + var attrs = {}; + for (var i = 0; i < el.attributes.length; i++) { + attrs[el.attributes[i].name] = el.attributes[i].value; + } + var rect = el.getBoundingClientRect(); + var result = { + tag: el.tagName.toLowerCase(), + text: (el.innerText || '').substring(0, 500), + selector: buildSelector(el), + attributes: attrs, + url: location.href, + outerHTML: el.outerHTML.substring(0, 2000), + computedStyles: getRelevantStyles(el), + __paseoSessionToken: sessionToken, + boundingRect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, + reactSource: getReactSource(el), + parentChain: getParentChain(el, 5), + children: getChildSummary(el, 8) + }; + destroy(); + window.__paseoSelectorResult = result; + } + function onKey(e) { + if (e.key === 'Escape') { + destroy(); + window.__paseoSelectorResult = { __cancelled: true, __paseoSessionToken: sessionToken }; + } + } + function blockEvent(e) { + e.preventDefault(); + e.stopPropagation(); + e.stopImmediatePropagation(); + } + function destroy() { + document.removeEventListener('mousemove', onMove, true); + document.removeEventListener('click', onClick, true); + document.removeEventListener('keydown', onKey, true); + document.removeEventListener('mousedown', blockEvent, true); + document.removeEventListener('mouseup', blockEvent, true); + document.removeEventListener('pointerdown', blockEvent, true); + document.removeEventListener('pointerup', blockEvent, true); + document.removeEventListener('touchstart', blockEvent, true); + document.removeEventListener('touchend', blockEvent, true); + document.removeEventListener('focus', blockEvent, true); + document.removeEventListener('submit', blockEvent, true); + document.documentElement.classList.remove('__paseo-select-mode'); + if (last) last.classList.remove('__paseo-hover'); + if (hoverLabel.parentNode) hoverLabel.parentNode.removeChild(hoverLabel); + style.remove(); + window.__paseoSelector = null; + } + document.addEventListener('mousemove', onMove, true); + document.addEventListener('click', onClick, true); + document.addEventListener('keydown', onKey, true); + document.addEventListener('mousedown', blockEvent, true); + document.addEventListener('mouseup', blockEvent, true); + document.addEventListener('pointerdown', blockEvent, true); + document.addEventListener('pointerup', blockEvent, true); + document.addEventListener('touchstart', blockEvent, true); + document.addEventListener('touchend', blockEvent, true); + document.addEventListener('focus', blockEvent, true); + document.addEventListener('submit', blockEvent, true); + window.__paseoSelector = { destroy: destroy, sessionToken: sessionToken }; + return { installed: true, sessionToken: sessionToken }; + })() + `; +} + +export function createElementSelectorController(): ElementSelectorController { + let currentSession: ElementSelectorSession | null = null; + let sessionCounter = 0; + + const finish = ( + session: ElementSelectorSession, + outcome: ElementSelectorOutcome, + guestCleanup: "clear" | "destroy" | null, + ): boolean => { + stopSessionTimers(session); + if (currentSession !== session) { + return false; + } + currentSession = null; + if (guestCleanup === "clear") { + clearWebviewSelector(session.webview, session.token); + } else if (guestCleanup === "destroy") { + destroyWebviewSelector(session.webview, session.token); + } + session.onFinish(outcome); + return true; + }; + + return { + start({ webview, mode, onFinish }) { + if (!webview.isConnected) { + return "unavailable"; + } + if (webview.isLoading?.()) { + return "loading"; + } + + if (currentSession) { + finish(currentSession, { type: "cancelled" }, "clear"); + } + + sessionCounter += 1; + const session: ElementSelectorSession = { + mode, + token: `${sessionCounter}:${crypto.randomUUID()}`, + webview, + onFinish, + }; + currentSession = session; + session.timeoutId = window.setTimeout(() => { + finish(session, { type: "failed", reason: "timeout" }, "destroy"); + }, SELECTOR_TIMEOUT_MS); + + void executeWebviewJavaScript(webview, buildElementSelectorScript(session.token)) + .then((installation) => { + const installationState = readSelectorInstallation(installation, session.token); + if (currentSession !== session) { + if (installationState === "installed") { + destroyWebviewSelector(webview, session.token); + } + return undefined; + } + if (installationState !== "installed") { + finish(session, { type: "failed", reason: installationState }, null); + return undefined; + } + session.stopPolling = startSelectorResultPolling({ + webview, + sessionToken: session.token, + onResult: (selection) => { + finish( + session, + selection + ? { type: "selected", mode: session.mode, selection } + : { type: "cancelled" }, + null, + ); + }, + }); + return undefined; + }) + .catch(() => { + finish(session, { type: "failed", reason: "unavailable" }, "destroy"); + return undefined; + }); + + return "started"; + }, + cancel() { + if (currentSession) { + finish(currentSession, { type: "cancelled" }, "clear"); + } + }, + stopForWebview(webview) { + if (currentSession?.webview === webview) { + finish(currentSession, { type: "cancelled" }, "destroy"); + } + }, + }; +} diff --git a/packages/app/src/desktop/browser/pane/index.electron.tsx b/packages/app/src/desktop/browser/pane/index.electron.tsx index a97e9ee92..4edc016ee 100644 --- a/packages/app/src/desktop/browser/pane/index.electron.tsx +++ b/packages/app/src/desktop/browser/pane/index.electron.tsx @@ -59,7 +59,6 @@ import { } from "@/desktop/browser/store"; import { applyInactiveBrowserWebviewViewport, - isResidentBrowserWebviewReady, prepareBrowserWebview, presentBrowserWebview, rememberBrowserWebviewSize, @@ -67,6 +66,11 @@ import { removeResidentBrowserWebview, takeResidentBrowserWebview, } from "../resident-webviews"; +import { + createElementSelectorController, + type BrowserElementSelection, + type ElementSelectorOutcome, +} from "./element-selector.electron"; type ElectronWebview = HTMLElement & { canGoBack?: () => boolean; @@ -77,6 +81,7 @@ type ElectronWebview = HTMLElement & { stop?: () => void; loadURL?: (url: string) => Promise; getURL?: () => string; + isLoading?: () => boolean; executeJavaScript?: (code: string) => Promise; focus?: () => void; addEventListener: (type: string, listener: EventListenerOrEventListenerObject) => void; @@ -87,10 +92,6 @@ type WebTextInput = TextInput & { getNativeRef?: () => unknown; }; -type BrowserElementSelection = Omit & { - attributes?: Record; -}; - interface BrowserElementAnnotation { comment: string; } @@ -315,20 +316,6 @@ function executeWebviewJavaScript(webview: ElectronWebview, code: string): Promi function ignoreWebviewJavaScriptError() {} -function destroyWebviewSelector(webview: ElectronWebview): void { - void executeWebviewJavaScript( - webview, - "if(window.__paseoSelector) window.__paseoSelector.destroy();", - ).catch(ignoreWebviewJavaScriptError); -} - -function clearWebviewSelector(webview: ElectronWebview): void { - void executeWebviewJavaScript( - webview, - "if(window.__paseoSelector) window.__paseoSelector.destroy(); window.__paseoSelectorResult = null;", - ).catch(ignoreWebviewJavaScriptError); -} - interface BrowserAnnotationMarker { index: number; selector: string; @@ -437,38 +424,6 @@ function isDesktopBrowserShortcutEvent(payload: unknown): payload is DesktopBrow return event.action === "focus-url"; } -function startSelectorResultPolling(input: { - webview: ElectronWebview; - onSelection: (selection: BrowserElementSelection) => void; - onDone: () => void; -}): number { - const { webview, onSelection, onDone } = input; - const poll = window.setInterval(() => { - void (async () => { - try { - const raw = await executeWebviewJavaScript( - webview, - "JSON.stringify(window.__paseoSelectorResult || null)", - ); - const result = typeof raw === "string" ? JSON.parse(raw) : null; - if (!result) { - return; - } - window.clearInterval(poll); - onDone(); - await executeWebviewJavaScript(webview, "window.__paseoSelectorResult = null;"); - if (!result.__cancelled) { - onSelection(result as BrowserElementSelection); - } - } catch { - // Keep polling; cross-origin/webview timing can make this transient. - } - })(); - }, 200); - - return poll; -} - function ToolbarButton({ label, children, @@ -655,13 +610,15 @@ export function BrowserPane({ const browserRef = useRef(browser); browserRef.current = browser; const pendingNavigationUrlRef = useRef(null); - const domReadyRef = useRef(false); const annotationMarkersRef = useRef([]); const [selectorMode, setSelectorMode] = useState<"annotate" | "screenshot" | null>(null); + const selectorControllerRef = useRef | null>( + null, + ); + if (!selectorControllerRef.current) { + selectorControllerRef.current = createElementSelectorController(); + } const selectorActive = selectorMode !== null; - // Which action the active selector performs on click: open the annotation card - // ("annotate") or copy a screenshot of the element to the clipboard ("screenshot"). - const selectorModeRef = useRef<"annotate" | "screenshot">("annotate"); const toast = useToast(); const toastRef = useRef(toast); toastRef.current = toast; @@ -737,7 +694,7 @@ export function BrowserPane({ const syncNavigationState = useCallback((input?: { syncUrl?: boolean }) => { const webview = webviewRef.current; - if (!webview || !domReadyRef.current) { + if (!webview) { return; } @@ -776,7 +733,6 @@ export function BrowserPane({ const residentWebview = takeResidentBrowserWebview(browserId) as ElectronWebview | null; const webview = residentWebview ?? (document.createElement("webview") as ElectronWebview); webviewRef.current = webview; - domReadyRef.current = isResidentBrowserWebviewReady(webview); if (!residentWebview) { prepareBrowserWebview(webview, { browserId, @@ -808,7 +764,7 @@ export function BrowserPane({ }); const handleStartLoading = () => { - domReadyRef.current = false; + selectorControllerRef.current?.stopForWebview(webview); updateBrowser(browserId, { isLoading: true, lastError: null }); syncNavigationState({ syncUrl: false }); }; @@ -875,7 +831,6 @@ export function BrowserPane({ }); }; const handleDomReady = () => { - domReadyRef.current = true; syncNavigationState(); // The previous page's overlay is gone after a load; re-apply markers for // the freshly loaded document. @@ -940,10 +895,10 @@ export function BrowserPane({ } else { removeResidentBrowserWebview(browserIdRef.current); } + selectorControllerRef.current?.stopForWebview(webview); if (webviewRef.current === webview) { webviewRef.current = null; } - domReadyRef.current = false; }; // eslint-disable-next-line react-hooks/exhaustive-deps }, [browserId, onFocusPane]); @@ -1196,8 +1151,8 @@ export function BrowserPane({ ); const handleSelectorResult = useCallback( - (selection: BrowserElementSelection) => { - if (selectorModeRef.current === "screenshot") { + (selection: BrowserElementSelection, mode: "annotate" | "screenshot") => { + if (mode === "screenshot") { void screenshotElementToClipboard(selection); return; } @@ -1230,271 +1185,62 @@ export function BrowserPane({ setPendingSelection(null); }, []); + const showSelectorFailure = useCallback( + (reason: "loading" | "timeout" | "unavailable") => { + const message = + reason === "loading" + ? t("workspace.browser.controls.selectorLoading") + : t("workspace.browser.controls.selectorFailed"); + toastRef.current?.error(message); + }, + [t], + ); + + const handleElementSelectorOutcome = useCallback( + (outcome: ElementSelectorOutcome) => { + setSelectorMode(null); + if (outcome.type === "selected") { + handleSelectorResult(outcome.selection, outcome.mode); + } else if (outcome.type === "failed") { + showSelectorFailure(outcome.reason); + } + }, + [handleSelectorResult, showSelectorFailure], + ); + const startElementSelector = useCallback( (mode: "annotate" | "screenshot") => { + if (mode === "annotate" && !workspaceAttachmentScopeKey) { + showSelectorFailure("unavailable"); + return; + } const webview = webviewRef.current; - if (!webview || !domReadyRef.current) return; - // Annotate needs a workspace scope to attach to; screenshot only copies. - if (mode === "annotate" && !workspaceAttachmentScopeKey) return; - selectorModeRef.current = mode; + const controller = selectorControllerRef.current; + if (!webview || !controller) { + showSelectorFailure("unavailable"); + return; + } + + const startResult = controller.start({ + webview, + mode, + onFinish: handleElementSelectorOutcome, + }); + if (startResult !== "started") { + showSelectorFailure(startResult); + return; + } + pendingScreenshotRef.current = undefined; setPendingSelection(null); setSelectorMode(mode); - - const js = ` - (function() { - if (window.__paseoSelector) { window.__paseoSelector.destroy(); } - var overlay = null; - var style = document.createElement('style'); - style.textContent = [ - '.__paseo-hover { outline: 2px solid #3b82f6 !important; outline-offset: 2px !important; cursor: crosshair !important; }', - '.__paseo-select-mode, .__paseo-select-mode * { cursor: crosshair !important; pointer-events: auto !important; user-select: none !important; }', - '.__paseo-select-mode *, .__paseo-select-mode *::before, .__paseo-select-mode *::after { animation: none !important; transition: none !important; }', - '.__paseo-select-mode a, .__paseo-select-mode button, .__paseo-select-mode input, .__paseo-select-mode select, .__paseo-select-mode textarea, .__paseo-select-mode [role="button"], .__paseo-select-mode [onclick] { pointer-events: none !important; }', - '.__paseo-select-mode iframe, .__paseo-select-mode video, .__paseo-select-mode audio { pointer-events: none !important; }', - '.__paseo-hover-label { position: fixed; z-index: 2147483647; pointer-events: none; max-width: 360px; padding: 4px 8px; border-radius: 6px; background: rgba(24,24,27,0.96); color: #fff; font: 500 11px/1.45 ui-monospace,SFMono-Regular,Menlo,monospace; box-shadow: 0 2px 10px rgba(0,0,0,0.35); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }', - '.__paseo-hover-label .__paseo-tag { color: #93c5fd; }', - '.__paseo-hover-label .__paseo-id { color: #fca5a5; }', - '.__paseo-hover-label .__paseo-cls { color: #fcd34d; }', - '.__paseo-hover-label .__paseo-dim { color: #a1a1aa; margin-left: 6px; }', - '.__paseo-hover-label .__paseo-comp { color: #86efac; margin-left: 6px; }', - ].join('\\n'); - document.head.appendChild(style); - document.documentElement.classList.add('__paseo-select-mode'); - var hoverLabel = document.createElement('div'); - hoverLabel.className = '__paseo-hover-label'; - hoverLabel.style.display = 'none'; - document.documentElement.appendChild(hoverLabel); - var last = null; - function escapeHtml(value) { - return String(value).replace(/[&<>"]/g, function(ch) { - return ch === '&' ? '&' : ch === '<' ? '<' : ch === '>' ? '>' : '"'; - }); - } - function describeElement(el) { - var tag = el.tagName ? el.tagName.toLowerCase() : 'node'; - var parts = ['' + escapeHtml(tag) + '']; - if (el.id) { - parts.push('#' + escapeHtml(el.id) + ''); - } - if (el.classList && el.classList.length) { - var cls = Array.prototype.slice.call(el.classList, 0, 2) - .filter(function(c) { return c.indexOf('__paseo') !== 0; }) - .map(function(c) { return '.' + escapeHtml(c); }) - .join(''); - if (cls) parts.push('' + cls + ''); - } - var comp = getReactSource(el); - if (comp && comp.componentName) { - parts.push('<' + escapeHtml(comp.componentName) + '>'); - } - var rect = el.getBoundingClientRect(); - parts.push('' + Math.round(rect.width) + '×' + Math.round(rect.height) + ''); - return { html: parts.join(''), rect: rect }; - } - function positionLabel(rect, e) { - var pad = 12; - var lw = hoverLabel.offsetWidth || 0; - var lh = hoverLabel.offsetHeight || 0; - var top = rect.top - lh - 6; - if (top < 4) top = rect.bottom + 6; - if (top + lh > window.innerHeight - 4) top = Math.max(4, e.clientY - lh - 6); - var left = rect.left; - if (left + lw > window.innerWidth - 4) left = Math.max(4, window.innerWidth - lw - 4); - if (left < 4) left = 4; - hoverLabel.style.top = Math.round(top) + 'px'; - hoverLabel.style.left = Math.round(left) + 'px'; - } - function onMove(e) { - e.preventDefault(); - e.stopPropagation(); - if (last) last.classList.remove('__paseo-hover'); - var el = e.target; - el.classList.add('__paseo-hover'); - last = el; - try { - var info = describeElement(el); - hoverLabel.innerHTML = info.html; - hoverLabel.style.display = 'block'; - positionLabel(info.rect, e); - } catch (err) { - hoverLabel.style.display = 'none'; - } - } - function buildSelector(el) { - if (el.id) return '#' + el.id; - var path = []; - while (el && el.nodeType === 1) { - var seg = el.tagName.toLowerCase(); - if (el.id) { path.unshift('#' + el.id); break; } - var sib = el, nth = 1; - while (sib = sib.previousElementSibling) { if (sib.tagName === el.tagName) nth++; } - if (nth > 1) seg += ':nth-of-type(' + nth + ')'; - path.unshift(seg); - el = el.parentElement; - } - return path.join(' > '); - } - function getReactSource(el) { - var keys = Object.keys(el); - for (var i = 0; i < keys.length; i++) { - if (keys[i].startsWith('__reactFiber$') || keys[i].startsWith('__reactInternalInstance$')) { - var fiber = el[keys[i]]; - while (fiber) { - if (fiber._debugSource) { - return { - fileName: fiber._debugSource.fileName || null, - lineNumber: fiber._debugSource.lineNumber || null, - columnNumber: fiber._debugSource.columnNumber || null, - componentName: (fiber.type && (typeof fiber.type === 'string' ? fiber.type : fiber.type.displayName || fiber.type.name)) || null - }; - } - if (fiber._debugOwner) { fiber = fiber._debugOwner; } - else if (fiber.return) { fiber = fiber.return; } - else break; - } - } - } - return null; - } - function getParentChain(el, depth) { - var chain = []; - var cur = el.parentElement; - for (var i = 0; i < (depth || 5) && cur; i++) { - var desc = cur.tagName.toLowerCase(); - if (cur.id) desc += '#' + cur.id; - if (cur.className && typeof cur.className === 'string') { var cls = cur.className.trim().replace(/ +/g, ' ').split(' ').slice(0,2).join('.'); if (cls) desc += '.' + cls; } - chain.push(desc); - cur = cur.parentElement; - } - return chain; - } - function getChildSummary(el, max) { - var kids = []; - for (var i = 0; i < Math.min(el.children.length, max || 8); i++) { - var c = el.children[i]; - var desc = c.tagName.toLowerCase(); - if (c.id) desc += '#' + c.id; - kids.push(desc); - } - if (el.children.length > (max || 8)) kids.push('...(' + el.children.length + ' total)'); - return kids; - } - function getRelevantStyles(el) { - var cs = window.getComputedStyle(el); - var pick = ['display','position','width','height','color','background-color','font-size','font-family','padding','margin','border','flex','grid-template-columns','gap','overflow','opacity','z-index']; - var out = {}; - pick.forEach(function(p) { - var v = cs.getPropertyValue(p); - if (v && v !== 'none' && v !== 'normal' && v !== 'auto' && v !== '0px' && v !== 'rgba(0, 0, 0, 0)') out[p] = v; - }); - return out; - } - function onClick(e) { - e.preventDefault(); - e.stopPropagation(); - e.stopImmediatePropagation(); - var el = e.target; - if (last) last.classList.remove('__paseo-hover'); - hoverLabel.style.display = 'none'; - var attrs = {}; - for (var i = 0; i < el.attributes.length; i++) { - attrs[el.attributes[i].name] = el.attributes[i].value; - } - var rect = el.getBoundingClientRect(); - var result = { - tag: el.tagName.toLowerCase(), - text: (el.innerText || '').substring(0, 500), - selector: buildSelector(el), - attributes: attrs, - url: location.href, - outerHTML: el.outerHTML.substring(0, 2000), - computedStyles: getRelevantStyles(el), - boundingRect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, - reactSource: getReactSource(el), - parentChain: getParentChain(el, 5), - children: getChildSummary(el, 8) - }; - destroy(); - window.__paseoSelectorResult = result; - } - function onKey(e) { - if (e.key === 'Escape') { destroy(); window.__paseoSelectorResult = { __cancelled: true }; } - } - function blockEvent(e) { - e.preventDefault(); - e.stopPropagation(); - e.stopImmediatePropagation(); - } - function destroy() { - document.removeEventListener('mousemove', onMove, true); - document.removeEventListener('click', onClick, true); - document.removeEventListener('keydown', onKey, true); - document.removeEventListener('mousedown', blockEvent, true); - document.removeEventListener('mouseup', blockEvent, true); - document.removeEventListener('pointerdown', blockEvent, true); - document.removeEventListener('pointerup', blockEvent, true); - document.removeEventListener('touchstart', blockEvent, true); - document.removeEventListener('touchend', blockEvent, true); - document.removeEventListener('focus', blockEvent, true); - document.removeEventListener('submit', blockEvent, true); - document.documentElement.classList.remove('__paseo-select-mode'); - if (last) last.classList.remove('__paseo-hover'); - if (hoverLabel.parentNode) hoverLabel.parentNode.removeChild(hoverLabel); - style.remove(); - window.__paseoSelector = null; - } - document.addEventListener('mousemove', onMove, true); - document.addEventListener('click', onClick, true); - document.addEventListener('keydown', onKey, true); - document.addEventListener('mousedown', blockEvent, true); - document.addEventListener('mouseup', blockEvent, true); - document.addEventListener('pointerdown', blockEvent, true); - document.addEventListener('pointerup', blockEvent, true); - document.addEventListener('touchstart', blockEvent, true); - document.addEventListener('touchend', blockEvent, true); - document.addEventListener('focus', blockEvent, true); - document.addEventListener('submit', blockEvent, true); - window.__paseoSelector = { destroy: destroy }; - })() - `; - - try { - void executeWebviewJavaScript(webview, js) - .then(() => { - const poll = startSelectorResultPolling({ - webview, - onSelection: handleSelectorResult, - onDone: () => setSelectorMode(null), - }); - window.setTimeout(() => { - window.clearInterval(poll); - setSelectorMode(null); - if (webviewRef.current !== webview || !domReadyRef.current) { - return; - } - destroyWebviewSelector(webview); - }, 30000); - return undefined; - }) - .catch(() => { - setSelectorMode(null); - }); - } catch { - setSelectorMode(null); - } }, - [handleSelectorResult, workspaceAttachmentScopeKey], + [handleElementSelectorOutcome, showSelectorFailure, workspaceAttachmentScopeKey], ); const cancelElementSelector = useCallback(() => { - const webview = webviewRef.current; + selectorControllerRef.current?.cancel(); setSelectorMode(null); - if (webview && domReadyRef.current) { - try { - clearWebviewSelector(webview); - } catch {} - } }, []); const currentPageUrl = browser?.url ?? null; @@ -1526,7 +1272,7 @@ export function BrowserPane({ return; } const webview = webviewRef.current; - if (!webview || !domReadyRef.current) { + if (!webview) { return; } if (annotationMarkers.length === 0) { diff --git a/packages/app/src/desktop/browser/resident-webviews.browser.test.ts b/packages/app/src/desktop/browser/resident-webviews.browser.test.ts index 868005817..679081df0 100644 --- a/packages/app/src/desktop/browser/resident-webviews.browser.test.ts +++ b/packages/app/src/desktop/browser/resident-webviews.browser.test.ts @@ -5,7 +5,6 @@ import { clearResidentBrowserWebviewsForTests, ensureResidentBrowserWebview, getResidentBrowserWebview, - isResidentBrowserWebviewReady, prepareBrowserWebview, presentBrowserWebview, rememberBrowserWebviewSize, @@ -205,28 +204,6 @@ describe("resident browser webviews", () => { expect(webview.parentElement.style.height).toBe("600px"); }); - it("retains document readiness with the resident webview", () => { - const webview = ensureTestBrowser({ - browserId: "browser-ready", - workspaceId: "workspace-ready", - url: "https://example.com", - }); - if (!webview) { - throw new Error("Expected resident browser webview"); - } - - expect(isResidentBrowserWebviewReady(webview)).toBe(false); - webview.dispatchEvent(new Event("dom-ready")); - releaseResidentBrowserWebview("browser-ready", webview); - - const reusedWebview = takeResidentBrowserWebview("browser-ready"); - expect(reusedWebview).toBe(webview); - expect(isResidentBrowserWebviewReady(reusedWebview as HTMLElement)).toBe(true); - - webview.dispatchEvent(new Event("did-start-loading")); - expect(isResidentBrowserWebviewReady(webview)).toBe(false); - }); - it("creates a resident webview for an agent-created unfocused tab", () => { const webview = ensureTestBrowser({ browserId: "browser-agent", diff --git a/packages/app/src/desktop/browser/resident-webviews.ts b/packages/app/src/desktop/browser/resident-webviews.ts index 0b5352584..dfce3551e 100644 --- a/packages/app/src/desktop/browser/resident-webviews.ts +++ b/packages/app/src/desktop/browser/resident-webviews.ts @@ -15,7 +15,6 @@ const RESIDENT_VIEWPORT_HEIGHT = 800; const residentWebviewsByBrowserId = new Map(); const residentSurfacesByBrowserId = new Map(); const residentWebviewSizesByBrowserId = new Map(); -const readyResidentWebviews = new WeakSet(); interface BrowserWebviewElement extends HTMLElement { src: string; @@ -75,15 +74,6 @@ function registerBrowserWhenAttached( }); } -function registerBrowserReadiness(webview: HTMLElement): void { - webview.addEventListener("did-start-loading", () => { - readyResidentWebviews.delete(webview); - }); - webview.addEventListener("dom-ready", () => { - readyResidentWebviews.add(webview); - }); -} - function trimNonEmpty(value: string | null | undefined): string | null { if (typeof value !== "string") { return null; @@ -322,11 +312,6 @@ export function prepareBrowserWebview( (webview as BrowserWebviewElement).src = input.initialUrl; } registerBrowserWhenAttached(webview as BrowserWebviewElement, input, browser); - registerBrowserReadiness(webview); -} - -export function isResidentBrowserWebviewReady(webview: HTMLElement): boolean { - return readyResidentWebviews.has(webview); } export function ensureResidentBrowserWebview(input: { diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index f72bcb1a3..0247a2ba4 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -523,6 +523,8 @@ export const ar: TranslationResources = { screenshotCopied: "تم نسخ لقطة الشاشة إلى الحافظة", elementCopied: "تم نسخ العنصر إلى الحافظة", screenshotFailed: "تعذّر نسخ لقطة الشاشة", + selectorLoading: "انتظر حتى ينتهي تحميل الصفحة", + selectorFailed: "تعذّر بدء محدد العنصر", }, annotate: { title: "التعليق على العنصر", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 83ff3c8b3..b655cd397 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -522,6 +522,8 @@ export const en = { screenshotCopied: "Copied screenshot to clipboard", elementCopied: "Copied element to clipboard", screenshotFailed: "Couldn't copy screenshot", + selectorLoading: "Wait for the page to finish loading", + selectorFailed: "Couldn't start element selector", }, annotate: { title: "Annotate element", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index 4d38d6e70..911c3977f 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -527,6 +527,8 @@ export const es: TranslationResources = { screenshotCopied: "Captura copiada al portapapeles", elementCopied: "Elemento copiado al portapapeles", screenshotFailed: "No se pudo copiar la captura", + selectorLoading: "Espera a que la página termine de cargar", + selectorFailed: "No se pudo iniciar el selector de elementos", }, annotate: { title: "Anotar elemento", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 44c61c65e..ac39cbd04 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -527,6 +527,8 @@ export const fr: TranslationResources = { screenshotCopied: "Capture d'écran copiée dans le presse-papiers", elementCopied: "Élément copié dans le presse-papiers", screenshotFailed: "Impossible de copier la capture", + selectorLoading: "Attendez la fin du chargement de la page", + selectorFailed: "Impossible de démarrer le sélecteur d’élément", }, annotate: { title: "Annoter l'élément", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index 6d2eaace6..915bb6aaf 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -527,6 +527,8 @@ export const ja: TranslationResources = { screenshotCopied: "スクリーンショットをクリップボードにコピーしました", elementCopied: "要素をクリップボードにコピーしました", screenshotFailed: "スクリーンショットをコピーできませんでした", + selectorLoading: "ページの読み込みが完了するまでお待ちください", + selectorFailed: "要素セレクターを開始できませんでした", }, annotate: { title: "要素に注釈を付ける", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index f7e76a6c5..17bb85dbb 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -524,6 +524,8 @@ export const ko: TranslationResources = { screenshotCopied: "스크린샷을 클립보드에 복사했습니다.", elementCopied: "요소를 클립보드에 복사했습니다.", screenshotFailed: "스크린샷을 복사할 수 없습니다.", + selectorLoading: "페이지 로딩이 끝날 때까지 기다려 주세요.", + selectorFailed: "요소 선택기를 시작할 수 없습니다.", }, annotate: { title: "요소에 주석 달기", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 5006b2251..fea908d4b 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -527,6 +527,8 @@ export const ptBR: TranslationResources = { screenshotCopied: "Captura copiada para a área de transferência", elementCopied: "Elemento copiado para a área de transferência", screenshotFailed: "Não foi possível copiar a captura", + selectorLoading: "Aguarde o carregamento da página terminar", + selectorFailed: "Não foi possível iniciar o seletor de elemento", }, annotate: { title: "Anotar elemento", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index 0bd5ccbbe..3305774cb 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -527,6 +527,8 @@ export const ru: TranslationResources = { screenshotCopied: "Снимок скопирован в буфер обмена", elementCopied: "Элемент скопирован в буфер обмена", screenshotFailed: "Не удалось скопировать снимок", + selectorLoading: "Дождитесь окончания загрузки страницы", + selectorFailed: "Не удалось запустить выбор элемента", }, annotate: { title: "Аннотировать элемент", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index fffc9dc6a..bc02aa5bf 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -523,6 +523,8 @@ export const zhCN: TranslationResources = { screenshotCopied: "已将截图复制到剪贴板", elementCopied: "已将元素复制到剪贴板", screenshotFailed: "无法复制截图", + selectorLoading: "请等待页面加载完成", + selectorFailed: "无法启动元素选择器", }, annotate: { title: "标注元素", diff --git a/packages/desktop/e2e/browser-tabs.e2e.mjs b/packages/desktop/e2e/browser-tabs.e2e.mjs index 6c8ec44d7..7ff2bee92 100644 --- a/packages/desktop/e2e/browser-tabs.e2e.mjs +++ b/packages/desktop/e2e/browser-tabs.e2e.mjs @@ -232,6 +232,21 @@ async function callBrowserTool(client, name, args = {}) { return mcpPayload(await client.callTool({ name, args }), name); } +async function waitForGuestSelector(client, browserId) { + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + const evaluated = await callBrowserTool(client, "browser_evaluate", { + browserId, + function: "() => Boolean(globalThis.__paseoSelector)", + }); + if (JSON.parse(evaluated.resultJson) === true) { + return true; + } + await delay(50); + } + return false; +} + async function createCallerAgent(daemonPort) { const transport = new StreamableHTTPClientTransport( new URL(`http://127.0.0.1:${daemonPort}/mcp/agents`), @@ -384,7 +399,7 @@ function recordViewportMismatch(failures, label, actual, expected) { ); } -async function runRegression({ page, client, serverId, targetUrl, callerAgentId }) { +async function runRegression({ page, client, serverId, targetUrl, callerAgentId, artifactDir }) { const failures = []; const originalWorkspaceId = workspaceIds[0]; const originalWorkspaceRow = page.getByTestId( @@ -673,16 +688,90 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId responsiveViewport, ); - await originalDeck.getByRole("button", { name: "Annotate element" }).click(); - await page.waitForTimeout(250); - const selectorResult = await callBrowserTool(client, "browser_evaluate", { + const annotateButton = originalDeck.getByRole("button", { name: "Annotate element" }); + await page.evaluate((id) => { + const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`); + if (!(webview instanceof HTMLElement)) throw new Error(`Browser webview ${id} was unavailable`); + globalThis.__paseoOriginalIsLoadingDescriptor = Object.getOwnPropertyDescriptor( + webview, + "isLoading", + ); + Object.defineProperty(webview, "isLoading", { + configurable: true, + value: () => true, + }); + }, browserId); + await annotateButton.click(); + await page.waitForTimeout(100); + assert( + !(await originalDeck.getByRole("button", { name: "Cancel element selector" }).isVisible()), + "Element selector started while the guest reported a genuine load", + ); + const selectorDuringLoad = await callBrowserTool(client, "browser_evaluate", { browserId, function: "() => Boolean(globalThis.__paseoSelector)", }); - if (JSON.parse(selectorResult.resultJson) !== true) { - failures.push("reused loaded browser remains ready for element annotation after remount"); + assert( + JSON.parse(selectorDuringLoad.resultJson) === false, + "Selector injected while the guest reported a genuine load", + ); + assert( + await page.getByText("Wait for the page to finish loading").isVisible(), + "Element selector loading failure was not visible", + ); + await page.evaluate((id) => { + const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`); + if (!(webview instanceof HTMLElement)) throw new Error(`Browser webview ${id} was unavailable`); + const descriptor = globalThis.__paseoOriginalIsLoadingDescriptor; + if (descriptor) Object.defineProperty(webview, "isLoading", descriptor); + else delete webview.isLoading; + delete globalThis.__paseoOriginalIsLoadingDescriptor; + }, browserId); + + const readyStateResult = await callBrowserTool(client, "browser_evaluate", { + browserId, + function: "() => document.readyState", + }); + assert( + JSON.parse(readyStateResult.resultJson) === "complete", + "Local browser document did not finish loading", + ); + // Reproduce the report's mismatch: the guest is complete, but the pane's + // last loading signal says it is not ready. + await page.evaluate((id) => { + const webview = document.querySelector(`[data-paseo-browser-id="${id}"]`); + if (!(webview instanceof HTMLElement)) { + throw new Error(`Browser webview ${id} was unavailable`); + } + webview.dispatchEvent(new Event("did-start-loading")); + }, browserId); + + await annotateButton.click(); + const annotateSelectorActive = await waitForGuestSelector(client, browserId); + await page.screenshot({ path: path.join(artifactDir, "local-page-annotate-selector.png") }); + if (!annotateSelectorActive) { + failures.push("loaded local browser remains ready for element annotation"); } + await originalDeck.getByRole("button", { name: "Cancel element selector" }).click(); + await delay(10_000); + const screenshotButton = originalDeck.getByRole("button", { name: "Screenshot element" }); + await screenshotButton.click(); + const screenshotSelectorActive = await waitForGuestSelector(client, browserId); + if (!screenshotSelectorActive) { + failures.push("loaded local browser remains ready for element screenshots"); + } + await delay(20_500); + const selectorAfterPriorTimeout = await callBrowserTool(client, "browser_evaluate", { + browserId, + function: "() => Boolean(globalThis.__paseoSelector)", + }); + if (JSON.parse(selectorAfterPriorTimeout.resultJson) !== true) { + failures.push("a previous selector timeout does not destroy the current selector session"); + } + await page.screenshot({ path: path.join(artifactDir, "local-page-screenshot-selector.png") }); + await originalDeck.getByRole("button", { name: "Cancel element selector" }).click(); + if (failures.length > 0) { throw new Error(`Browser viewport regressions:\n- ${failures.join("\n- ")}`); } @@ -698,6 +787,7 @@ async function runRegression({ page, client, serverId, targetUrl, callerAgentId list: "passed", snapshot: "passed", click: "passed", + localPageSelectors: "passed", }; } @@ -796,10 +886,11 @@ async function main() { serverId: status.serverId, targetUrl: target.url, callerAgentId, + artifactDir, }); writeJson(path.join(artifactDir, "result.json"), report); console.log( - `Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click passed.`, + `Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click, local-page selectors passed.`, ); } catch (error) { console.error(`Browser desktop browser E2E failed. Artifacts: ${artifactDir}`);