Browser tools: full accessibility snapshot, trusted input, dialogs, evaluate, and tab controls (#1881)
* feat(browser): replace flat snapshot with ARIA tree and page-side refs * feat(browser): trusted CDP input with actionability waits * feat(browser): auto-handle javascript dialogs * feat(browser): add browser_evaluate * feat(browser): add scroll, resize, and close_tab tools * fix(browser): review fixes for snapshot/input/dialog seams * fix(browser): address review findings on dialog capture, truncation, and keypress targets * fix(harness): drop resize check that cannot run under parked compositor surfaces * fix(browser): second review round — snapshot fidelity, key input, dialog scoping
This commit is contained in:
Mohamed Boudra
@@ -1,24 +1,60 @@
|
||||
import { ARIA_SNAPSHOT_SCRIPT, ARIA_SNAPSHOT_SCRIPT_MARKER } from "./aria-snapshot-script.js";
|
||||
|
||||
export interface SnapshotPage {
|
||||
getURL(): string;
|
||||
executeJavaScript(code: string): Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface BrowserSnapshotElement extends RawSnapshotElement {
|
||||
ref: string;
|
||||
export interface BrowserAriaSnapshot {
|
||||
format: "aria-yaml";
|
||||
snapshot: string;
|
||||
truncated: boolean;
|
||||
stats: BrowserAriaSnapshotStats;
|
||||
}
|
||||
|
||||
interface RawSnapshotElement {
|
||||
export interface BrowserAriaSnapshotStats {
|
||||
nodeCount: number;
|
||||
refCount: number;
|
||||
textLength: number;
|
||||
iframeCount?: number;
|
||||
maxDepth?: number;
|
||||
}
|
||||
|
||||
interface SnapshotNode {
|
||||
kind: "role" | "text" | "group";
|
||||
role?: string;
|
||||
name?: string;
|
||||
text?: string;
|
||||
tagName?: string;
|
||||
attributes?: string[];
|
||||
ref?: string;
|
||||
fingerprint?: BrowserRefFingerprint;
|
||||
children?: SnapshotNode[];
|
||||
}
|
||||
|
||||
interface BrowserRefFingerprint {
|
||||
role: string;
|
||||
name: string;
|
||||
tagName: string;
|
||||
text: string;
|
||||
selector: string;
|
||||
attributes: Record<string, string>;
|
||||
type: string;
|
||||
ariaLabel: string;
|
||||
}
|
||||
|
||||
interface RawAriaSnapshot {
|
||||
marker: string;
|
||||
root: SnapshotNode;
|
||||
refs: BrowserRefMetadata[];
|
||||
truncated: boolean;
|
||||
stats: BrowserAriaSnapshotStats;
|
||||
}
|
||||
|
||||
interface BrowserRefMetadata {
|
||||
ref: string;
|
||||
fingerprint: BrowserRefFingerprint;
|
||||
}
|
||||
|
||||
interface BrowserRefState {
|
||||
nextRefNumber: number;
|
||||
url: string;
|
||||
refs: Map<string, RawSnapshotElement>;
|
||||
refs: Map<string, BrowserRefMetadata>;
|
||||
}
|
||||
|
||||
export type BrowserRefActionResult =
|
||||
@@ -26,45 +62,31 @@ export type BrowserRefActionResult =
|
||||
| { ok: false; reason: "stale_ref" | "missing_ref" };
|
||||
type BrowserRefFailure = Extract<BrowserRefActionResult, { ok: false }>;
|
||||
|
||||
type BrowserRefResolveResult = { ok: true; element: RawSnapshotElement } | BrowserRefFailure;
|
||||
type BrowserRefResolveResult = { ok: true; metadata: BrowserRefMetadata } | BrowserRefFailure;
|
||||
|
||||
const TRUNCATION_MARKER = '- text: "Snapshot truncated."';
|
||||
const MAX_RENDERED_TEXT_LENGTH = 80_000;
|
||||
|
||||
export class BrowserSnapshotEngine {
|
||||
private readonly statesByBrowserId = new Map<string, BrowserRefState>();
|
||||
|
||||
async snapshot(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
}): Promise<BrowserSnapshotElement[]> {
|
||||
const rawElements = parseRawSnapshotElements(
|
||||
await input.page.executeJavaScript(SNAPSHOT_SCRIPT),
|
||||
);
|
||||
const state = {
|
||||
nextRefNumber: 1,
|
||||
url: input.page.getURL(),
|
||||
refs: new Map<string, RawSnapshotElement>(),
|
||||
};
|
||||
const elements = rawElements.map((element) => {
|
||||
const ref = `@e${state.nextRefNumber++}`;
|
||||
state.refs.set(ref, element);
|
||||
return {
|
||||
ref,
|
||||
role: element.role,
|
||||
tagName: element.tagName,
|
||||
text: element.text,
|
||||
selector: element.selector,
|
||||
attributes: element.attributes,
|
||||
};
|
||||
async snapshot(input: { browserId: string; page: SnapshotPage }): Promise<BrowserAriaSnapshot> {
|
||||
const rawSnapshot = parseAriaSnapshot(await input.page.executeJavaScript(ARIA_SNAPSHOT_SCRIPT));
|
||||
const rendered = renderSnapshot(rawSnapshot.root);
|
||||
const capped = capRenderedSnapshot(rendered, rawSnapshot.truncated);
|
||||
this.statesByBrowserId.set(input.browserId, {
|
||||
refs: new Map(rawSnapshot.refs.map((ref) => [ref.ref, ref])),
|
||||
});
|
||||
this.statesByBrowserId.set(input.browserId, state);
|
||||
return elements;
|
||||
}
|
||||
|
||||
async click(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildClickScript(selector));
|
||||
return {
|
||||
format: "aria-yaml",
|
||||
snapshot: capped.snapshot,
|
||||
truncated: capped.truncated,
|
||||
stats: {
|
||||
...rawSnapshot.stats,
|
||||
refCount: rawSnapshot.refs.length,
|
||||
textLength: capped.snapshot.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async fill(input: {
|
||||
@@ -73,39 +95,7 @@ export class BrowserSnapshotEngine {
|
||||
ref: string;
|
||||
value: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildFillScript(selector, input.value));
|
||||
}
|
||||
|
||||
async typeText(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref?: string;
|
||||
text: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
const selector = this.resolveOptionalRef(input);
|
||||
if (!selector.ok) {
|
||||
return selector;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildTypeScript(selector.selector, input.text),
|
||||
);
|
||||
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
}
|
||||
|
||||
async keypress(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref?: string;
|
||||
key: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
const selector = this.resolveOptionalRef(input);
|
||||
if (!selector.ok) {
|
||||
return selector;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildKeypressScript(selector.selector, input.key),
|
||||
);
|
||||
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
return this.runRefScript(input, (ref) => buildFillScript(ref, input.value));
|
||||
}
|
||||
|
||||
async select(input: {
|
||||
@@ -114,123 +104,214 @@ export class BrowserSnapshotEngine {
|
||||
ref: string;
|
||||
value: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildSelectScript(selector, input.value));
|
||||
}
|
||||
|
||||
async hover(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildHoverScript(selector));
|
||||
}
|
||||
|
||||
async drag(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
sourceRef: string;
|
||||
targetRef: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
const source = this.resolveRef({
|
||||
browserId: input.browserId,
|
||||
page: input.page,
|
||||
ref: input.sourceRef,
|
||||
});
|
||||
if (!source.ok) {
|
||||
return source;
|
||||
}
|
||||
const target = this.resolveRef({
|
||||
browserId: input.browserId,
|
||||
page: input.page,
|
||||
ref: input.targetRef,
|
||||
});
|
||||
if (!target.ok) {
|
||||
return target;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildDragScript(source.element.selector, target.element.selector),
|
||||
);
|
||||
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
return this.runRefScript(input, (ref) => buildSelectScript(ref, input.value));
|
||||
}
|
||||
|
||||
clearBrowser(browserId: string): void {
|
||||
this.statesByBrowserId.delete(browserId);
|
||||
}
|
||||
|
||||
selectorForRef(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): { ok: true; selector: string } | BrowserRefFailure {
|
||||
runtimeElementExpression(input: { browserId: string; ref: string }): string | BrowserRefFailure {
|
||||
const resolved = this.resolveRef(input);
|
||||
if (!resolved.ok) {
|
||||
return resolved;
|
||||
}
|
||||
return { ok: true, selector: resolved.element.selector };
|
||||
return buildRuntimeElementExpression(resolved.metadata);
|
||||
}
|
||||
|
||||
private async runRefScript(
|
||||
input: { browserId: string; page: SnapshotPage; ref: string },
|
||||
buildScript: (selector: string) => string,
|
||||
buildScript: (ref: BrowserRefMetadata) => string,
|
||||
): Promise<BrowserRefActionResult> {
|
||||
const resolved = this.resolveRef(input);
|
||||
if (!resolved.ok) {
|
||||
return resolved;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(buildScript(resolved.element.selector));
|
||||
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
const result = await input.page.executeJavaScript(buildScript(resolved.metadata));
|
||||
return readActionResult(result, true);
|
||||
}
|
||||
|
||||
private resolveRef(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): BrowserRefResolveResult {
|
||||
private resolveRef(input: { browserId: string; ref: string }): BrowserRefResolveResult {
|
||||
const state = this.statesByBrowserId.get(input.browserId);
|
||||
if (!state || state.url !== input.page.getURL()) {
|
||||
if (!state) {
|
||||
return { ok: false, reason: "stale_ref" };
|
||||
}
|
||||
const element = state.refs.get(input.ref);
|
||||
if (!element) {
|
||||
const metadata = state.refs.get(input.ref);
|
||||
if (!metadata) {
|
||||
return { ok: false, reason: "missing_ref" };
|
||||
}
|
||||
return { ok: true, element };
|
||||
}
|
||||
|
||||
private resolveOptionalRef(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref?: string;
|
||||
}): { ok: true; selector: string | undefined } | BrowserRefFailure {
|
||||
if (!input.ref) {
|
||||
return { ok: true, selector: undefined };
|
||||
}
|
||||
const resolved = this.resolveRef({
|
||||
browserId: input.browserId,
|
||||
page: input.page,
|
||||
ref: input.ref,
|
||||
});
|
||||
if (!resolved.ok) {
|
||||
return resolved;
|
||||
}
|
||||
return { ok: true, selector: resolved.element.selector };
|
||||
return { ok: true, metadata };
|
||||
}
|
||||
}
|
||||
|
||||
function buildClickScript(selector: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
element.click();
|
||||
return true;
|
||||
})()`;
|
||||
function readActionResult(value: unknown, staleWhenFalse: boolean): BrowserRefActionResult {
|
||||
if (value === false && staleWhenFalse) {
|
||||
return { ok: false, reason: "stale_ref" };
|
||||
}
|
||||
if (isActionFailure(value)) {
|
||||
return { ok: false, reason: value.reason };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
function buildFillScript(selector: string, value: string): string {
|
||||
function isActionFailure(value: unknown): value is BrowserRefFailure {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const reason = (value as Record<string, unknown>).reason;
|
||||
return reason === "stale_ref" || reason === "missing_ref";
|
||||
}
|
||||
|
||||
function parseAriaSnapshot(value: unknown): RawAriaSnapshot {
|
||||
const parsed = typeof value === "string" ? JSON.parse(value) : value;
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
return emptySnapshot();
|
||||
}
|
||||
const record = parsed as Record<string, unknown>;
|
||||
if (record.marker !== ARIA_SNAPSHOT_SCRIPT_MARKER) {
|
||||
return emptySnapshot();
|
||||
}
|
||||
const root = parseSnapshotNode(record.root);
|
||||
return {
|
||||
marker: ARIA_SNAPSHOT_SCRIPT_MARKER,
|
||||
root: root ?? emptySnapshot().root,
|
||||
refs: parseRefs(record.refs),
|
||||
truncated: record.truncated === true,
|
||||
stats: parseStats(record.stats),
|
||||
};
|
||||
}
|
||||
|
||||
function emptySnapshot(): RawAriaSnapshot {
|
||||
return {
|
||||
marker: ARIA_SNAPSHOT_SCRIPT_MARKER,
|
||||
root: { kind: "role", role: "document", name: "", tagName: "document", children: [] },
|
||||
refs: [],
|
||||
truncated: false,
|
||||
stats: { nodeCount: 0, refCount: 0, textLength: 0, iframeCount: 0, maxDepth: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
function parseSnapshotNode(value: unknown): SnapshotNode | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
const kind = record.kind;
|
||||
if (kind !== "role" && kind !== "text" && kind !== "group") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
kind,
|
||||
...(readString(record.role) ? { role: readString(record.role) ?? undefined } : {}),
|
||||
...(readString(record.name) ? { name: readString(record.name) ?? undefined } : {}),
|
||||
...(readString(record.text) ? { text: readString(record.text) ?? undefined } : {}),
|
||||
...(readString(record.tagName) ? { tagName: readString(record.tagName) ?? undefined } : {}),
|
||||
...(readString(record.ref) ? { ref: readString(record.ref) ?? undefined } : {}),
|
||||
...(parseFingerprint(record.fingerprint)
|
||||
? { fingerprint: parseFingerprint(record.fingerprint) ?? undefined }
|
||||
: {}),
|
||||
attributes: readStringArray(record.attributes),
|
||||
children: Array.isArray(record.children)
|
||||
? record.children.flatMap((child): SnapshotNode[] => {
|
||||
const parsed = parseSnapshotNode(child);
|
||||
return parsed ? [parsed] : [];
|
||||
})
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
function parseRefs(value: unknown): BrowserRefMetadata[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.flatMap((item): BrowserRefMetadata[] => {
|
||||
if (!item || typeof item !== "object") {
|
||||
return [];
|
||||
}
|
||||
const record = item as Record<string, unknown>;
|
||||
const ref = readString(record.ref);
|
||||
const fingerprint = parseFingerprint(record.fingerprint);
|
||||
return ref && fingerprint ? [{ ref, fingerprint }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
function parseFingerprint(value: unknown): BrowserRefFingerprint | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
const role = readString(record.role);
|
||||
const name = readString(record.name);
|
||||
const tagName = readString(record.tagName);
|
||||
const type = readString(record.type);
|
||||
const ariaLabel = readString(record.ariaLabel);
|
||||
if (role === null || name === null || tagName === null || type === null || ariaLabel === null) {
|
||||
return null;
|
||||
}
|
||||
return { role, name, tagName, type, ariaLabel };
|
||||
}
|
||||
|
||||
function parseStats(value: unknown): BrowserAriaSnapshotStats {
|
||||
if (!value || typeof value !== "object") {
|
||||
return { nodeCount: 0, refCount: 0, textLength: 0 };
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return {
|
||||
nodeCount: readNumber(record.nodeCount) ?? 0,
|
||||
refCount: readNumber(record.refCount) ?? 0,
|
||||
textLength: readNumber(record.textLength) ?? 0,
|
||||
...(readNumber(record.iframeCount) !== null
|
||||
? { iframeCount: readNumber(record.iframeCount) ?? undefined }
|
||||
: {}),
|
||||
...(readNumber(record.maxDepth) !== null
|
||||
? { maxDepth: readNumber(record.maxDepth) ?? undefined }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function renderSnapshot(root: SnapshotNode): string {
|
||||
const lines = renderNode(root, 0);
|
||||
return lines.length > 0 ? lines.join("\n") : "- document";
|
||||
}
|
||||
|
||||
function renderNode(node: SnapshotNode, depth: number): string[] {
|
||||
if (node.kind === "group") {
|
||||
return (node.children ?? []).flatMap((child) => renderNode(child, depth));
|
||||
}
|
||||
const indent = " ".repeat(depth);
|
||||
if (node.kind === "text") {
|
||||
return [`${indent}- text: ${JSON.stringify(node.text ?? "")}`];
|
||||
}
|
||||
const attrs = [...(node.attributes ?? [])];
|
||||
if (node.ref) {
|
||||
attrs.push(`ref=${node.ref}`);
|
||||
}
|
||||
const suffix = attrs.length > 0 ? ` [${attrs.join(" ")}]` : "";
|
||||
const ownLine = `${indent}- ${node.role ?? "generic"}${node.name ? ` ${JSON.stringify(node.name)}` : ""}${suffix}`;
|
||||
const childLines = (node.children ?? []).flatMap((child) => renderNode(child, depth + 1));
|
||||
return [ownLine, ...childLines];
|
||||
}
|
||||
|
||||
function capRenderedSnapshot(
|
||||
snapshot: string,
|
||||
alreadyTruncated: boolean,
|
||||
): { snapshot: string; truncated: boolean } {
|
||||
if (snapshot.length <= MAX_RENDERED_TEXT_LENGTH && !alreadyTruncated) {
|
||||
return { snapshot, truncated: false };
|
||||
}
|
||||
if (snapshot.length + 1 + TRUNCATION_MARKER.length <= MAX_RENDERED_TEXT_LENGTH) {
|
||||
return { snapshot: `${snapshot}\n${TRUNCATION_MARKER}`, truncated: true };
|
||||
}
|
||||
const availableLength = MAX_RENDERED_TEXT_LENGTH - TRUNCATION_MARKER.length - 1;
|
||||
const capped = snapshot.slice(0, Math.max(0, availableLength)).replace(/\n[^\n]*$/, "");
|
||||
return { snapshot: `${capped}\n${TRUNCATION_MARKER}`, truncated: true };
|
||||
}
|
||||
|
||||
function buildFillScript(metadata: BrowserRefMetadata, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
const resolved = ${buildResolveExpression(metadata)};
|
||||
if (!resolved.ok) return resolved;
|
||||
const element = resolved.element;
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
element.focus();
|
||||
const nextValue = ${JSON.stringify(value)};
|
||||
@@ -238,51 +319,19 @@ function buildFillScript(selector: string, value: string): string {
|
||||
element.value = nextValue;
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
return { ok: true };
|
||||
}
|
||||
element.textContent = nextValue;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: nextValue }));
|
||||
return true;
|
||||
return { ok: true };
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildTypeScript(selector: string | undefined, text: string): string {
|
||||
function buildSelectScript(metadata: BrowserRefMetadata, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
|
||||
if (!element) return false;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
element.focus?.();
|
||||
const text = ${JSON.stringify(text)};
|
||||
if ('value' in element) {
|
||||
element.value = String(element.value || '') + text;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
}
|
||||
element.textContent = String(element.textContent || '') + text;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildKeypressScript(selector: string | undefined, key: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
|
||||
if (!element) return false;
|
||||
element.focus?.();
|
||||
const key = ${JSON.stringify(key)};
|
||||
const eventInit = { bubbles: true, cancelable: true, key };
|
||||
element.dispatchEvent(new KeyboardEvent('keydown', eventInit));
|
||||
element.dispatchEvent(new KeyboardEvent('keypress', eventInit));
|
||||
element.dispatchEvent(new KeyboardEvent('keyup', eventInit));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildSelectScript(selector: string, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
const resolved = ${buildResolveExpression(metadata)};
|
||||
if (!resolved.ok) return resolved;
|
||||
const element = resolved.element;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
element.focus?.();
|
||||
const nextValue = ${JSON.stringify(value)};
|
||||
@@ -290,205 +339,34 @@ function buildSelectScript(selector: string, value: string): string {
|
||||
element.value = nextValue;
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
return { ok: true };
|
||||
}
|
||||
return false;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildHoverScript(selector: string): string {
|
||||
function buildRuntimeElementExpression(metadata: BrowserRefMetadata): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
const rect = element.getBoundingClientRect();
|
||||
const eventInit = {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
clientX: rect.left + rect.width / 2,
|
||||
clientY: rect.top + rect.height / 2,
|
||||
screenX: window.screenX + rect.left + rect.width / 2,
|
||||
screenY: window.screenY + rect.top + rect.height / 2,
|
||||
view: window,
|
||||
};
|
||||
element.dispatchEvent(new MouseEvent('mouseover', eventInit));
|
||||
element.dispatchEvent(new MouseEvent('mouseenter', eventInit));
|
||||
element.dispatchEvent(new MouseEvent('mousemove', eventInit));
|
||||
return true;
|
||||
const resolved = ${buildResolveExpression(metadata)};
|
||||
if (!resolved.ok) return null;
|
||||
return resolved.element;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildDragScript(sourceSelector: string, targetSelector: string): string {
|
||||
return String.raw`(() => {
|
||||
const source = document.querySelector(${JSON.stringify(sourceSelector)});
|
||||
const target = document.querySelector(${JSON.stringify(targetSelector)});
|
||||
if (!source || !target) return false;
|
||||
source.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
target.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
const data = new DataTransfer();
|
||||
const sourceRect = source.getBoundingClientRect();
|
||||
const targetRect = target.getBoundingClientRect();
|
||||
function eventInit(rect) {
|
||||
return {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
clientX: rect.left + rect.width / 2,
|
||||
clientY: rect.top + rect.height / 2,
|
||||
screenX: window.screenX + rect.left + rect.width / 2,
|
||||
screenY: window.screenY + rect.top + rect.height / 2,
|
||||
dataTransfer: data,
|
||||
view: window,
|
||||
};
|
||||
}
|
||||
source.dispatchEvent(new MouseEvent('mousedown', eventInit(sourceRect)));
|
||||
source.dispatchEvent(new DragEvent('dragstart', eventInit(sourceRect)));
|
||||
target.dispatchEvent(new DragEvent('dragenter', eventInit(targetRect)));
|
||||
target.dispatchEvent(new DragEvent('dragover', eventInit(targetRect)));
|
||||
target.dispatchEvent(new DragEvent('drop', eventInit(targetRect)));
|
||||
source.dispatchEvent(new DragEvent('dragend', eventInit(sourceRect)));
|
||||
target.dispatchEvent(new MouseEvent('mouseup', eventInit(targetRect)));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function parseRawSnapshotElements(value: unknown): RawSnapshotElement[] {
|
||||
const parsed = typeof value === "string" ? JSON.parse(value) : value;
|
||||
if (!Array.isArray(parsed)) {
|
||||
return [];
|
||||
}
|
||||
return parsed.flatMap((item): RawSnapshotElement[] => {
|
||||
if (!item || typeof item !== "object") {
|
||||
return [];
|
||||
}
|
||||
const record = item as Record<string, unknown>;
|
||||
const selector = readString(record.selector);
|
||||
if (!selector) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
role: readString(record.role) || "generic",
|
||||
tagName: (readString(record.tagName) || "element").toLowerCase(),
|
||||
text: readString(record.text) || "",
|
||||
selector,
|
||||
attributes: readAttributes(record.attributes),
|
||||
},
|
||||
];
|
||||
});
|
||||
function buildResolveExpression(metadata: BrowserRefMetadata): string {
|
||||
return `window.__PASEO_BROWSER_AUTOMATION__?.resolve(${JSON.stringify(metadata.ref)}, ${JSON.stringify(metadata.fingerprint)}) ?? { ok: false, reason: 'stale_ref' }`;
|
||||
}
|
||||
|
||||
function readString(value: unknown): string | null {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
function readAttributes(value: unknown): Record<string, string> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return {};
|
||||
}
|
||||
const result: Record<string, string> = {};
|
||||
for (const [key, attributeValue] of Object.entries(value)) {
|
||||
if (typeof attributeValue === "string") {
|
||||
result[key] = attributeValue;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
function readStringArray(value: unknown): string[] {
|
||||
return Array.isArray(value)
|
||||
? value.filter((item): item is string => typeof item === "string")
|
||||
: [];
|
||||
}
|
||||
|
||||
const SNAPSHOT_SCRIPT = String.raw`(() => {
|
||||
const MAX_ELEMENTS = 200;
|
||||
const CANDIDATE_SELECTOR = [
|
||||
'a[href]',
|
||||
'button',
|
||||
'input',
|
||||
'textarea',
|
||||
'select',
|
||||
'summary',
|
||||
'[role]',
|
||||
'[tabindex]:not([tabindex="-1"])',
|
||||
'[contenteditable=""]',
|
||||
'[contenteditable="true"]'
|
||||
].join(',');
|
||||
|
||||
function cssEscape(value) {
|
||||
if (window.CSS && typeof window.CSS.escape === 'function') {
|
||||
return window.CSS.escape(value);
|
||||
}
|
||||
return String(value).replace(/[^a-zA-Z0-9_-]/g, '\\$&');
|
||||
}
|
||||
|
||||
function isVisible(element) {
|
||||
const style = window.getComputedStyle(element);
|
||||
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
|
||||
return false;
|
||||
}
|
||||
const rect = element.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}
|
||||
|
||||
function roleFor(element) {
|
||||
const explicit = element.getAttribute('role');
|
||||
if (explicit) return explicit;
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (tag === 'a') return 'link';
|
||||
if (tag === 'button') return 'button';
|
||||
if (tag === 'select') return 'combobox';
|
||||
if (tag === 'textarea') return 'textbox';
|
||||
if (tag === 'summary') return 'button';
|
||||
if (tag === 'input') {
|
||||
const type = (element.getAttribute('type') || 'text').toLowerCase();
|
||||
if (type === 'checkbox') return 'checkbox';
|
||||
if (type === 'radio') return 'radio';
|
||||
if (type === 'button' || type === 'submit' || type === 'reset') return 'button';
|
||||
return 'textbox';
|
||||
}
|
||||
return 'generic';
|
||||
}
|
||||
|
||||
function textFor(element) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
const pieces = [
|
||||
element.getAttribute('aria-label'),
|
||||
element.getAttribute('alt'),
|
||||
element.getAttribute('title'),
|
||||
tag === 'input' ? element.getAttribute('placeholder') : null,
|
||||
tag === 'input' || tag === 'textarea' ? element.value : null,
|
||||
element.innerText,
|
||||
element.textContent
|
||||
];
|
||||
const text = pieces.find((piece) => typeof piece === 'string' && piece.trim().length > 0);
|
||||
return (text || '').replace(/\s+/g, ' ').trim().slice(0, 300);
|
||||
}
|
||||
|
||||
function selectorFor(element) {
|
||||
if (element.id) return '#' + cssEscape(element.id);
|
||||
const parts = [];
|
||||
let current = element;
|
||||
while (current && current.nodeType === Node.ELEMENT_NODE && current !== document.body) {
|
||||
const tag = current.tagName.toLowerCase();
|
||||
const parent = current.parentElement;
|
||||
if (!parent) break;
|
||||
const siblings = Array.from(parent.children).filter((sibling) => sibling.tagName === current.tagName);
|
||||
const index = siblings.indexOf(current) + 1;
|
||||
parts.unshift(siblings.length > 1 ? tag + ':nth-of-type(' + index + ')' : tag);
|
||||
current = parent;
|
||||
}
|
||||
return parts.length > 0 ? parts.join(' > ') : element.tagName.toLowerCase();
|
||||
}
|
||||
|
||||
return JSON.stringify(Array.from(document.querySelectorAll(CANDIDATE_SELECTOR))
|
||||
.filter(isVisible)
|
||||
.slice(0, MAX_ELEMENTS)
|
||||
.map((element) => ({
|
||||
role: roleFor(element),
|
||||
tagName: element.tagName.toLowerCase(),
|
||||
text: textFor(element),
|
||||
selector: selectorFor(element),
|
||||
attributes: {
|
||||
...(element.id ? { id: element.id } : {}),
|
||||
...(element.getAttribute('name') ? { name: element.getAttribute('name') } : {}),
|
||||
...(element.getAttribute('type') ? { type: element.getAttribute('type') } : {}),
|
||||
...(element.getAttribute('href') ? { href: element.getAttribute('href') } : {}),
|
||||
...(element.getAttribute('aria-label') ? { 'aria-label': element.getAttribute('aria-label') } : {})
|
||||
}
|
||||
})));
|
||||
})()`;
|
||||
function readNumber(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user