Browser tools: full accessibility snapshot, trusted input, dialogs, evaluate, and tab controls (#1881)
* feat(browser): replace flat snapshot with ARIA tree and page-side refs * feat(browser): trusted CDP input with actionability waits * feat(browser): auto-handle javascript dialogs * feat(browser): add browser_evaluate * feat(browser): add scroll, resize, and close_tab tools * fix(browser): review fixes for snapshot/input/dialog seams * fix(browser): address review findings on dialog capture, truncation, and keypress targets * fix(harness): drop resize check that cannot run under parked compositor surfaces * fix(browser): second review round — snapshot fidelity, key input, dialog scoping
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,198 @@
|
||||
import type { SnapshotPage } from "./snapshot-engine.js";
|
||||
|
||||
export interface ActionablePoint {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface ActionableTarget {
|
||||
point: ActionablePoint;
|
||||
rect: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
}
|
||||
|
||||
export type ActionabilityResult =
|
||||
| { ok: true; target: ActionableTarget }
|
||||
| { ok: false; reason: "stale_ref" | "timeout"; detail?: string };
|
||||
|
||||
const DEFAULT_ACTIONABILITY_TIMEOUT_MS = 5_000;
|
||||
|
||||
export async function waitForActionableTarget(input: {
|
||||
page: SnapshotPage;
|
||||
elementExpression: string;
|
||||
editable?: boolean;
|
||||
timeoutMs?: number;
|
||||
}): Promise<ActionabilityResult> {
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildActionabilityScript({
|
||||
elementExpression: input.elementExpression,
|
||||
editable: input.editable === true,
|
||||
timeoutMs: input.timeoutMs ?? DEFAULT_ACTIONABILITY_TIMEOUT_MS,
|
||||
}),
|
||||
);
|
||||
return readActionabilityResult(result);
|
||||
}
|
||||
|
||||
function readActionabilityResult(value: unknown): ActionabilityResult {
|
||||
if (!value || typeof value !== "object") {
|
||||
return { ok: false, reason: "timeout" };
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
if (record.ok === true && isActionableTarget(record.target)) {
|
||||
return { ok: true, target: record.target };
|
||||
}
|
||||
if (record.ok === false) {
|
||||
const reason = record.reason;
|
||||
if (reason === "stale_ref" || reason === "timeout") {
|
||||
return {
|
||||
ok: false,
|
||||
reason,
|
||||
...(typeof record.detail === "string" ? { detail: record.detail } : {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
return { ok: false, reason: "timeout" };
|
||||
}
|
||||
|
||||
function isActionableTarget(value: unknown): value is ActionableTarget {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return isPoint(record.point) && isRect(record.rect);
|
||||
}
|
||||
|
||||
function isPoint(value: unknown): value is ActionablePoint {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return isFiniteNumber(record.x) && isFiniteNumber(record.y);
|
||||
}
|
||||
|
||||
function isRect(value: unknown): value is ActionableTarget["rect"] {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return (
|
||||
isFiniteNumber(record.x) &&
|
||||
isFiniteNumber(record.y) &&
|
||||
isFiniteNumber(record.width) &&
|
||||
isFiniteNumber(record.height)
|
||||
);
|
||||
}
|
||||
|
||||
function isFiniteNumber(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
function buildActionabilityScript(input: {
|
||||
elementExpression: string;
|
||||
editable: boolean;
|
||||
timeoutMs: number;
|
||||
}): string {
|
||||
return String.raw`(async () => {
|
||||
const deadline = performance.now() + ${JSON.stringify(input.timeoutMs)};
|
||||
const requiresEditable = ${JSON.stringify(input.editable)};
|
||||
|
||||
const nextFrame = () => new Promise((resolve) => requestAnimationFrame(resolve));
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const nearlyEqual = (a, b) => Math.abs(a - b) < 0.25;
|
||||
const sameRect = (a, b) =>
|
||||
nearlyEqual(a.x, b.x) &&
|
||||
nearlyEqual(a.y, b.y) &&
|
||||
nearlyEqual(a.width, b.width) &&
|
||||
nearlyEqual(a.height, b.height);
|
||||
const rectPayload = (rect) => ({
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
});
|
||||
const centerPoint = (rect) => ({
|
||||
x: Math.min(Math.max(rect.left + rect.width / 2, 0), Math.max(window.innerWidth - 1, 0)),
|
||||
y: Math.min(Math.max(rect.top + rect.height / 2, 0), Math.max(window.innerHeight - 1, 0)),
|
||||
});
|
||||
const isDisabled = (element) => {
|
||||
if (element.closest?.('[aria-disabled="true"]')) return true;
|
||||
if ('disabled' in element && element.disabled) return true;
|
||||
const fieldset = element.closest?.('fieldset[disabled]');
|
||||
return Boolean(fieldset);
|
||||
};
|
||||
const isEditable = (element) => {
|
||||
if (element.isContentEditable) return true;
|
||||
const tag = element.tagName?.toLowerCase();
|
||||
if (tag === 'textarea' || tag === 'select') return !element.readOnly && !isDisabled(element);
|
||||
if (tag !== 'input') return false;
|
||||
const type = (element.getAttribute('type') || 'text').toLowerCase();
|
||||
if (['button', 'checkbox', 'color', 'file', 'hidden', 'image', 'radio', 'range', 'reset', 'submit'].includes(type)) return false;
|
||||
return !element.readOnly && !isDisabled(element);
|
||||
};
|
||||
const isVisible = (element, rect) => {
|
||||
const style = getComputedStyle(element);
|
||||
return (
|
||||
rect.width > 0 &&
|
||||
rect.height > 0 &&
|
||||
style.visibility !== 'hidden' &&
|
||||
style.display !== 'none' &&
|
||||
Number(style.opacity || '1') !== 0
|
||||
);
|
||||
};
|
||||
const hitTargetReceivesEvents = (element, point) => {
|
||||
const hit = document.elementFromPoint(point.x, point.y);
|
||||
return Boolean(hit && (hit === element || element.contains(hit)));
|
||||
};
|
||||
const resolveElement = () => (${input.elementExpression});
|
||||
|
||||
let detail = 'not actionable';
|
||||
while (performance.now() <= deadline) {
|
||||
const element = resolveElement();
|
||||
if (!element || !element.isConnected) {
|
||||
return { ok: false, reason: 'stale_ref', detail: 'ref no longer resolves' };
|
||||
}
|
||||
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (!isVisible(element, rect)) {
|
||||
detail = 'not visible';
|
||||
await sleep(25);
|
||||
continue;
|
||||
}
|
||||
if (isDisabled(element)) {
|
||||
detail = 'disabled';
|
||||
await sleep(25);
|
||||
continue;
|
||||
}
|
||||
if (requiresEditable && !isEditable(element)) {
|
||||
detail = 'not editable';
|
||||
await sleep(25);
|
||||
continue;
|
||||
}
|
||||
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
await nextFrame();
|
||||
const firstRect = element.getBoundingClientRect();
|
||||
await nextFrame();
|
||||
const secondRect = element.getBoundingClientRect();
|
||||
if (!sameRect(firstRect, secondRect)) {
|
||||
detail = 'moving';
|
||||
continue;
|
||||
}
|
||||
|
||||
const point = centerPoint(secondRect);
|
||||
if (!hitTargetReceivesEvents(element, point)) {
|
||||
detail = 'covered';
|
||||
await sleep(25);
|
||||
continue;
|
||||
}
|
||||
|
||||
return { ok: true, target: { point, rect: rectPayload(secondRect) } };
|
||||
}
|
||||
|
||||
return { ok: false, reason: 'timeout', detail };
|
||||
})()`;
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
export const ARIA_SNAPSHOT_SCRIPT_MARKER = "__PASEO_ARIA_SNAPSHOT__";
|
||||
|
||||
// Adapted from Playwright's injected ARIA snapshot model.
|
||||
// Copyright (c) Microsoft Corporation. Licensed under the Apache License, Version 2.0.
|
||||
export const ARIA_SNAPSHOT_SCRIPT = String.raw`(() => {
|
||||
const MARKER = ${JSON.stringify(ARIA_SNAPSHOT_SCRIPT_MARKER)};
|
||||
const MAX_NODES = 1500;
|
||||
const MAX_REFS = 500;
|
||||
const MAX_TEXT_LENGTH = 80000;
|
||||
const ACTIONABLE_ROLES = new Set([
|
||||
'button',
|
||||
'checkbox',
|
||||
'combobox',
|
||||
'link',
|
||||
'menuitem',
|
||||
'option',
|
||||
'radio',
|
||||
'searchbox',
|
||||
'slider',
|
||||
'spinbutton',
|
||||
'switch',
|
||||
'tab',
|
||||
'textbox',
|
||||
'treeitem'
|
||||
]);
|
||||
|
||||
function normalizeText(value) {
|
||||
return String(value || '').replace(/[\u200b\u00ad]/g, '').replace(/[\r\n\s\t]+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function visibilityFor(element) {
|
||||
if (!(element instanceof Element)) return false;
|
||||
const style = window.getComputedStyle(element);
|
||||
if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0 ? 'box' : 'boxless';
|
||||
}
|
||||
|
||||
function explicitRole(element) {
|
||||
const role = element.getAttribute('role');
|
||||
if (!role || role === 'presentation' || role === 'none') return null;
|
||||
return role.split(/\s+/)[0].toLowerCase();
|
||||
}
|
||||
|
||||
function implicitRole(element) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (/^h[1-6]$/.test(tag)) return 'heading';
|
||||
if (tag === 'a' && element.hasAttribute('href')) return 'link';
|
||||
if (tag === 'button') return 'button';
|
||||
if (tag === 'select') return 'combobox';
|
||||
if (tag === 'textarea') return 'textbox';
|
||||
if (element instanceof HTMLElement && element.isContentEditable) return 'textbox';
|
||||
if (tag === 'summary') return 'button';
|
||||
if (tag === 'main') return 'main';
|
||||
if (tag === 'nav') return 'navigation';
|
||||
if (tag === 'header') return 'banner';
|
||||
if (tag === 'footer') return 'contentinfo';
|
||||
if (tag === 'section') return element.getAttribute('aria-label') ? 'region' : null;
|
||||
if (tag === 'ul' || tag === 'ol') return 'list';
|
||||
if (tag === 'li') return 'listitem';
|
||||
if (tag === 'table') return 'table';
|
||||
if (tag === 'tr') return 'row';
|
||||
if (tag === 'th') return 'columnheader';
|
||||
if (tag === 'td') return 'cell';
|
||||
if (tag === 'iframe') return 'iframe';
|
||||
if (tag === 'input') {
|
||||
const type = (element.getAttribute('type') || 'text').toLowerCase();
|
||||
if (type === 'checkbox') return 'checkbox';
|
||||
if (type === 'radio') return 'radio';
|
||||
if (type === 'button' || type === 'submit' || type === 'reset') return 'button';
|
||||
if (type === 'range') return 'slider';
|
||||
if (type === 'number') return 'spinbutton';
|
||||
if (type === 'search') return 'searchbox';
|
||||
if (type === 'hidden') return null;
|
||||
return 'textbox';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function roleFor(element) {
|
||||
return explicitRole(element) || implicitRole(element);
|
||||
}
|
||||
|
||||
function labelText(element) {
|
||||
if (!(element instanceof HTMLElement)) return '';
|
||||
if (element.id) {
|
||||
const escapedId = window.CSS && typeof window.CSS.escape === 'function'
|
||||
? window.CSS.escape(element.id)
|
||||
: String(element.id).replace(/"/g, '\\"');
|
||||
const label = document.querySelector('label[for="' + escapedId + '"]');
|
||||
if (label) return normalizeText(label.textContent);
|
||||
}
|
||||
const closestLabel = element.closest('label');
|
||||
return closestLabel ? normalizeText(closestLabel.textContent) : '';
|
||||
}
|
||||
|
||||
function nameFor(element, role) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
const labelledBy = element.getAttribute('aria-labelledby');
|
||||
if (labelledBy) {
|
||||
const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.textContent || '').join(' ');
|
||||
const normalized = normalizeText(text);
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
const pieces = [
|
||||
element.getAttribute('aria-label'),
|
||||
labelText(element),
|
||||
element.getAttribute('alt'),
|
||||
element.getAttribute('title'),
|
||||
tag === 'input' || tag === 'textarea' ? element.getAttribute('placeholder') : null,
|
||||
tag === 'input' || tag === 'textarea' ? element.value : null,
|
||||
role === 'button' || role === 'link' || role === 'heading' || ACTIONABLE_ROLES.has(role)
|
||||
? element.textContent
|
||||
: null
|
||||
];
|
||||
return normalizeText(pieces.find((piece) => normalizeText(piece).length > 0) || '');
|
||||
}
|
||||
|
||||
function fingerprintNameFor(element, role, name) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (tag !== 'input' && tag !== 'textarea') return name;
|
||||
const mutableValue = normalizeText(element.value);
|
||||
if (!mutableValue || name !== mutableValue) return name;
|
||||
return '';
|
||||
}
|
||||
|
||||
function inheritedDisabled(element) {
|
||||
if (!(element instanceof Element)) return false;
|
||||
if (element.hasAttribute('disabled') || element.getAttribute('aria-disabled') === 'true') return true;
|
||||
if (element.closest('fieldset[disabled]')) return true;
|
||||
return element.closest('[aria-disabled="true"]') !== null;
|
||||
}
|
||||
|
||||
function isActionable(element, role) {
|
||||
if (!role || !ACTIONABLE_ROLES.has(role)) return false;
|
||||
if (visibilityFor(element) !== 'box') return false;
|
||||
if (inheritedDisabled(element)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function fingerprintFor(element, role, name) {
|
||||
return {
|
||||
role,
|
||||
name: fingerprintNameFor(element, role, name),
|
||||
tagName: element.tagName.toLowerCase(),
|
||||
type: element.getAttribute('type') || '',
|
||||
ariaLabel: element.getAttribute('aria-label') || ''
|
||||
};
|
||||
}
|
||||
|
||||
function fingerprintMatches(element, fingerprint) {
|
||||
const role = roleFor(element) || 'generic';
|
||||
const name = nameFor(element, role);
|
||||
const current = fingerprintFor(element, role, name);
|
||||
return current.role === fingerprint.role &&
|
||||
current.name === fingerprint.name &&
|
||||
current.tagName === fingerprint.tagName &&
|
||||
current.type === fingerprint.type &&
|
||||
current.ariaLabel === fingerprint.ariaLabel;
|
||||
}
|
||||
|
||||
function ensureRuntime() {
|
||||
const runtime = {
|
||||
refs: new Map(),
|
||||
resolve(ref, fingerprint) {
|
||||
const element = this.refs.get(ref);
|
||||
if (!element || !element.isConnected || !fingerprintMatches(element, fingerprint)) {
|
||||
return { ok: false, reason: 'stale_ref' };
|
||||
}
|
||||
return { ok: true, element };
|
||||
}
|
||||
};
|
||||
Object.defineProperty(window, '__PASEO_BROWSER_AUTOMATION__', {
|
||||
configurable: true,
|
||||
enumerable: false,
|
||||
value: runtime
|
||||
});
|
||||
return runtime;
|
||||
}
|
||||
|
||||
function stateAttributes(element, role) {
|
||||
const attrs = [];
|
||||
if (role === 'heading') {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
const level = /^h[1-6]$/.test(tag) ? Number(tag.slice(1)) : Number(element.getAttribute('aria-level') || 0);
|
||||
if (level) attrs.push('level=' + level);
|
||||
}
|
||||
if (element.matches?.('input[type="checkbox"], input[type="radio"]')) {
|
||||
attrs.push('checked=' + (element.checked ? 'true' : 'false'));
|
||||
}
|
||||
if (element.getAttribute('aria-checked')) attrs.push('checked=' + element.getAttribute('aria-checked'));
|
||||
if (element.getAttribute('aria-expanded')) attrs.push('expanded=' + element.getAttribute('aria-expanded'));
|
||||
if (element.getAttribute('aria-pressed')) attrs.push('pressed=' + element.getAttribute('aria-pressed'));
|
||||
if (element.getAttribute('aria-selected')) attrs.push('selected=' + element.getAttribute('aria-selected'));
|
||||
return attrs;
|
||||
}
|
||||
|
||||
function textNode(text) {
|
||||
return { kind: 'text', text };
|
||||
}
|
||||
|
||||
function elementNode(element, role, name) {
|
||||
return {
|
||||
kind: 'role',
|
||||
role: role || 'generic',
|
||||
name,
|
||||
tagName: element.tagName.toLowerCase(),
|
||||
attributes: stateAttributes(element, role),
|
||||
children: []
|
||||
};
|
||||
}
|
||||
|
||||
let nodeCount = 0;
|
||||
let refCount = 0;
|
||||
let iframeCount = 0;
|
||||
let maxDepth = 0;
|
||||
let truncated = false;
|
||||
let textBudget = MAX_TEXT_LENGTH;
|
||||
const runtime = ensureRuntime();
|
||||
|
||||
function countNode(depth) {
|
||||
if (nodeCount >= MAX_NODES) {
|
||||
truncated = true;
|
||||
return false;
|
||||
}
|
||||
nodeCount += 1;
|
||||
maxDepth = Math.max(maxDepth, depth);
|
||||
return true;
|
||||
}
|
||||
|
||||
function cappedText(text) {
|
||||
if (text.length <= textBudget) {
|
||||
textBudget -= text.length;
|
||||
return text;
|
||||
}
|
||||
truncated = true;
|
||||
const capped = text.slice(0, Math.max(0, textBudget));
|
||||
textBudget = 0;
|
||||
return capped;
|
||||
}
|
||||
|
||||
function visitNode(domNode, depth) {
|
||||
if (!countNode(depth)) return null;
|
||||
if (domNode.nodeType === Node.TEXT_NODE) {
|
||||
const text = cappedText(normalizeText(domNode.textContent));
|
||||
if (!text) return null;
|
||||
return textNode(text);
|
||||
}
|
||||
if (!(domNode instanceof Element)) return null;
|
||||
const visibility = visibilityFor(domNode);
|
||||
if (!visibility) return null;
|
||||
if (domNode.getAttribute('aria-hidden') === 'true') return null;
|
||||
|
||||
const role = roleFor(domNode);
|
||||
const name = role ? nameFor(domNode, role) : '';
|
||||
const children = [];
|
||||
for (const child of Array.from(domNode.childNodes)) {
|
||||
const childSnapshot = visitNode(child, depth + 1);
|
||||
if (childSnapshot) children.push(childSnapshot);
|
||||
if (truncated) break;
|
||||
}
|
||||
|
||||
if (domNode.tagName.toLowerCase() === 'iframe') {
|
||||
iframeCount += 1;
|
||||
}
|
||||
|
||||
if (visibility === 'boxless') {
|
||||
return children.length > 0 ? { kind: 'group', children } : null;
|
||||
}
|
||||
if (!role && children.length === 0) return null;
|
||||
const snapshotNode = role ? elementNode(domNode, role, name) : { kind: 'group', children: [] };
|
||||
snapshotNode.children = children;
|
||||
if (role && isActionable(domNode, role) && refCount < MAX_REFS) {
|
||||
const ref = '@e' + (refCount + 1);
|
||||
const fingerprint = fingerprintFor(domNode, role, name);
|
||||
refCount += 1;
|
||||
runtime.refs.set(ref, domNode);
|
||||
snapshotNode.ref = ref;
|
||||
snapshotNode.fingerprint = fingerprint;
|
||||
} else if (role && isActionable(domNode, role)) {
|
||||
truncated = true;
|
||||
}
|
||||
return snapshotNode;
|
||||
}
|
||||
|
||||
const root = {
|
||||
kind: 'role',
|
||||
role: 'document',
|
||||
name: normalizeText(document.title),
|
||||
tagName: 'document',
|
||||
attributes: [],
|
||||
children: []
|
||||
};
|
||||
for (const child of Array.from(document.body ? document.body.childNodes : document.documentElement.childNodes)) {
|
||||
const childSnapshot = visitNode(child, 1);
|
||||
if (childSnapshot) root.children.push(childSnapshot);
|
||||
if (truncated) break;
|
||||
}
|
||||
|
||||
return JSON.stringify({
|
||||
marker: MARKER,
|
||||
root,
|
||||
refs: Array.from(runtime.refs.entries()).map(([ref, element]) => {
|
||||
const role = roleFor(element) || 'generic';
|
||||
const name = nameFor(element, role);
|
||||
return { ref, fingerprint: fingerprintFor(element, role, name) };
|
||||
}),
|
||||
truncated,
|
||||
stats: { nodeCount, refCount, textLength: 0, iframeCount, maxDepth }
|
||||
});
|
||||
})()`;
|
||||
@@ -0,0 +1,28 @@
|
||||
export type CdpCommandSender = (
|
||||
command: string,
|
||||
params?: Record<string, unknown>,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export class CdpSessionQueue {
|
||||
private queue: Promise<void> = Promise.resolve();
|
||||
|
||||
public async run<T>(task: () => Promise<T>): Promise<T> {
|
||||
const previous = this.queue;
|
||||
let releaseCurrent = () => {};
|
||||
const current = new Promise<void>((resolve) => {
|
||||
releaseCurrent = resolve;
|
||||
});
|
||||
const tail = previous.catch(() => {}).then(() => current);
|
||||
this.queue = tail;
|
||||
|
||||
await previous.catch(() => {});
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
releaseCurrent();
|
||||
if (this.queue === tail) {
|
||||
this.queue = Promise.resolve();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import type { BrowserAutomationDialogEvent } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
|
||||
export const MAX_DIALOGS_PER_COMMAND = 20;
|
||||
|
||||
export interface JavaScriptDialogOpening {
|
||||
type?: unknown;
|
||||
message?: unknown;
|
||||
defaultPrompt?: unknown;
|
||||
}
|
||||
|
||||
interface DialogPolicy {
|
||||
readonly action: BrowserAutomationDialogEvent["action"];
|
||||
readonly accept: boolean;
|
||||
}
|
||||
|
||||
const DIALOG_POLICIES: Record<BrowserAutomationDialogEvent["type"], DialogPolicy> = {
|
||||
alert: { action: "accepted", accept: true },
|
||||
confirm: { action: "dismissed", accept: false },
|
||||
prompt: { action: "dismissed", accept: false },
|
||||
beforeunload: { action: "dismissed", accept: false },
|
||||
};
|
||||
|
||||
export function handledDialogEvent(opening: JavaScriptDialogOpening): BrowserAutomationDialogEvent {
|
||||
const type = normalizeDialogType(opening.type);
|
||||
const policy = DIALOG_POLICIES[type];
|
||||
return {
|
||||
type,
|
||||
message: typeof opening.message === "string" ? opening.message : String(opening.message ?? ""),
|
||||
...(typeof opening.defaultPrompt === "string" ? { defaultValue: opening.defaultPrompt } : {}),
|
||||
action: policy.action,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
export function dialogAcceptValue(type: BrowserAutomationDialogEvent["type"]): boolean {
|
||||
return DIALOG_POLICIES[type].accept;
|
||||
}
|
||||
|
||||
export function promptShimInstallScript(): string {
|
||||
return String.raw`(() => {
|
||||
const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__";
|
||||
const state = window[stateKey] || { prompts: [], installed: false };
|
||||
window[stateKey] = state;
|
||||
if (state.installed) return true;
|
||||
const originalPrompt = window.prompt;
|
||||
Object.defineProperty(state, "originalPrompt", { configurable: true, value: originalPrompt });
|
||||
const promptShim = (message = "", defaultValue = "") => {
|
||||
state.prompts.push({
|
||||
type: "prompt",
|
||||
message: String(message ?? ""),
|
||||
defaultValue: String(defaultValue ?? ""),
|
||||
action: "dismissed",
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
return null;
|
||||
};
|
||||
Object.defineProperty(state, "promptShim", { configurable: true, value: promptShim });
|
||||
window.prompt = promptShim;
|
||||
state.installed = true;
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function promptShimDrainScript(): string {
|
||||
return String.raw`(() => {
|
||||
const state = window.__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__;
|
||||
if (!state || !Array.isArray(state.prompts)) return [];
|
||||
return state.prompts.splice(0);
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function promptShimRestoreScript(): string {
|
||||
return String.raw`(() => {
|
||||
const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__";
|
||||
const state = window[stateKey];
|
||||
if (!state || !state.installed) return true;
|
||||
if (window.prompt === state.promptShim && typeof state.originalPrompt === "function") {
|
||||
window.prompt = state.originalPrompt;
|
||||
}
|
||||
delete window[stateKey];
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function normalizeDialogType(value: unknown): BrowserAutomationDialogEvent["type"] {
|
||||
return value === "alert" || value === "confirm" || value === "prompt" || value === "beforeunload"
|
||||
? value
|
||||
: "alert";
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Rectangle } from "electron";
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import type { TabImage } from "./service.js";
|
||||
import { adaptWebContents } from "./ipc.js";
|
||||
|
||||
@@ -16,6 +16,15 @@ class FakeImage implements TabImage {
|
||||
class FakeDebugger {
|
||||
public attachedProtocolVersions: string[] = [];
|
||||
public commands: Array<{ command: string; params: Record<string, unknown> }> = [];
|
||||
public blockCommands = false;
|
||||
public readonly blockedCommandNames = new Set<string>();
|
||||
public readonly failedCommandNames = new Set<string>();
|
||||
public readonly promptDialogs: unknown[] = [];
|
||||
public failPromptDrain = false;
|
||||
private messageListener:
|
||||
| ((event: unknown, method: string, params?: Record<string, unknown>) => void)
|
||||
| null = null;
|
||||
private readonly blockedCommands: Array<() => void> = [];
|
||||
|
||||
public isAttached(): boolean {
|
||||
return this.attachedProtocolVersions.length > 0;
|
||||
@@ -27,8 +36,48 @@ class FakeDebugger {
|
||||
|
||||
public async sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown> {
|
||||
this.commands.push({ command, params: params ?? {} });
|
||||
if (this.failedCommandNames.has(command)) {
|
||||
throw new Error(`${command} failed`);
|
||||
}
|
||||
if (this.blockCommands || this.blockedCommandNames.has(command)) {
|
||||
await new Promise<void>((resolve) => {
|
||||
this.blockedCommands.push(resolve);
|
||||
});
|
||||
}
|
||||
if (command === "Runtime.evaluate" && typeof params?.expression === "string") {
|
||||
if (params.expression.includes("state.prompts.splice(0)")) {
|
||||
if (this.failPromptDrain) {
|
||||
throw new Error("execution context destroyed");
|
||||
}
|
||||
return { result: { value: this.promptDialogs.splice(0) } };
|
||||
}
|
||||
return { result: { value: true } };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
public on(
|
||||
event: "message",
|
||||
listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
|
||||
): void {
|
||||
expect(event).toBe("message");
|
||||
this.messageListener = listener;
|
||||
}
|
||||
|
||||
public emitMessage(method: string, params?: Record<string, unknown>): void {
|
||||
if (!this.messageListener) {
|
||||
throw new Error("Debugger message listener was not registered");
|
||||
}
|
||||
this.messageListener({}, method, params);
|
||||
}
|
||||
|
||||
public finishNextCommand(): void {
|
||||
const resolve = this.blockedCommands.shift();
|
||||
if (!resolve) {
|
||||
throw new Error("No command is blocked");
|
||||
}
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
|
||||
type ConsoleMessageListener = (
|
||||
@@ -181,4 +230,323 @@ describe("browser automation IPC adapter", () => {
|
||||
{ command: "Page.captureScreenshot", params: { format: "png" } },
|
||||
]);
|
||||
});
|
||||
|
||||
test("serializes CDP commands per guest contents", async () => {
|
||||
const contents = new FakeWebContents(23);
|
||||
contents.debugger.blockCommands = true;
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
const first = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mouseMoved" });
|
||||
const second = tab.sendDebugCommand?.("Page.captureScreenshot", { format: "png" });
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mouseMoved" } },
|
||||
]);
|
||||
|
||||
contents.debugger.finishNextCommand();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mouseMoved" } },
|
||||
{ command: "Page.captureScreenshot", params: { format: "png" } },
|
||||
]);
|
||||
|
||||
contents.debugger.finishNextCommand();
|
||||
await expect(first).resolves.toEqual({ ok: true });
|
||||
await expect(second).resolves.toEqual({ ok: true });
|
||||
});
|
||||
|
||||
test("handles JavaScript dialogs through the per-tab CDP queue", async () => {
|
||||
const contents = new FakeWebContents(24);
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
const captured = tab.captureDialogs?.(async () => {
|
||||
contents.debugger.blockCommands = true;
|
||||
const input = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mouseReleased" });
|
||||
await flushMicrotasks();
|
||||
|
||||
contents.debugger.emitMessage("Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "Delete item?",
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Page.enable", params: {} },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mouseReleased" } },
|
||||
{ command: "Page.handleJavaScriptDialog", params: { accept: false } },
|
||||
]);
|
||||
|
||||
contents.debugger.blockCommands = false;
|
||||
contents.debugger.finishNextCommand();
|
||||
await input;
|
||||
await flushMicrotasks();
|
||||
return "done";
|
||||
});
|
||||
|
||||
await expect(captured).resolves.toEqual({
|
||||
result: "done",
|
||||
dialogs: [
|
||||
{
|
||||
type: "confirm",
|
||||
message: "Delete item?",
|
||||
action: "dismissed",
|
||||
timestamp: expect.any(Number),
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Page.enable", params: {} },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mouseReleased" } },
|
||||
{ command: "Page.handleJavaScriptDialog", params: { accept: false } },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles JavaScript dialogs while the triggering CDP input command is still in flight", async () => {
|
||||
const contents = new FakeWebContents(25);
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
const captured = tab.captureDialogs?.(async () => {
|
||||
contents.debugger.blockedCommandNames.add("Input.dispatchMouseEvent");
|
||||
const input = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mousePressed" });
|
||||
await flushMicrotasks();
|
||||
|
||||
contents.debugger.emitMessage("Page.javascriptDialogOpening", {
|
||||
type: "alert",
|
||||
message: "Saved",
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Page.enable", params: {} },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mousePressed" } },
|
||||
{ command: "Page.handleJavaScriptDialog", params: { accept: true } },
|
||||
]);
|
||||
|
||||
contents.debugger.blockedCommandNames.clear();
|
||||
contents.debugger.finishNextCommand();
|
||||
await input;
|
||||
return "done";
|
||||
});
|
||||
|
||||
await expect(captured).resolves.toEqual({
|
||||
result: "done",
|
||||
dialogs: [
|
||||
{
|
||||
type: "alert",
|
||||
message: "Saved",
|
||||
action: "accepted",
|
||||
timestamp: expect.any(Number),
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(contents.debugger.commands.at(-1)).toEqual({
|
||||
command: "Runtime.evaluate",
|
||||
params: {
|
||||
expression: expect.stringContaining("delete window[stateKey]"),
|
||||
returnByValue: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("keeps the prompt shim installed until overlapping captures finish", async () => {
|
||||
const contents = new FakeWebContents(26);
|
||||
const tab = adaptWebContents(contents);
|
||||
const firstStarted = deferred<void>();
|
||||
const secondStarted = deferred<void>();
|
||||
const finishFirst = deferred<void>();
|
||||
const finishSecond = deferred<void>();
|
||||
|
||||
const first = tab.captureDialogs?.(async () => {
|
||||
firstStarted.resolve();
|
||||
await finishFirst.promise;
|
||||
return "first";
|
||||
});
|
||||
await firstStarted.promise;
|
||||
|
||||
const second = tab.captureDialogs?.(async () => {
|
||||
secondStarted.resolve();
|
||||
await finishSecond.promise;
|
||||
return "second";
|
||||
});
|
||||
await secondStarted.promise;
|
||||
|
||||
contents.debugger.promptDialogs.push(
|
||||
{
|
||||
type: "prompt",
|
||||
message: "First?",
|
||||
defaultValue: "one",
|
||||
action: "dismissed",
|
||||
timestamp: 1,
|
||||
},
|
||||
{
|
||||
type: "prompt",
|
||||
message: "Second?",
|
||||
defaultValue: "two",
|
||||
action: "dismissed",
|
||||
timestamp: 2,
|
||||
},
|
||||
);
|
||||
|
||||
finishFirst.resolve();
|
||||
await expect(first).resolves.toEqual({
|
||||
result: "first",
|
||||
dialogs: [
|
||||
{
|
||||
type: "prompt",
|
||||
message: "First?",
|
||||
defaultValue: "one",
|
||||
action: "dismissed",
|
||||
timestamp: 1,
|
||||
},
|
||||
{
|
||||
type: "prompt",
|
||||
message: "Second?",
|
||||
defaultValue: "two",
|
||||
action: "dismissed",
|
||||
timestamp: 2,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(
|
||||
contents.debugger.commands.some(
|
||||
(entry) =>
|
||||
entry.command === "Runtime.evaluate" &&
|
||||
typeof entry.params.expression === "string" &&
|
||||
entry.params.expression.includes("delete window[stateKey]"),
|
||||
),
|
||||
).toBe(false);
|
||||
|
||||
finishSecond.resolve();
|
||||
await expect(second).resolves.toEqual({
|
||||
result: "second",
|
||||
dialogs: [
|
||||
{
|
||||
type: "prompt",
|
||||
message: "First?",
|
||||
defaultValue: "one",
|
||||
action: "dismissed",
|
||||
timestamp: 1,
|
||||
},
|
||||
{
|
||||
type: "prompt",
|
||||
message: "Second?",
|
||||
defaultValue: "two",
|
||||
action: "dismissed",
|
||||
timestamp: 2,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(contents.debugger.commands.at(-1)).toEqual({
|
||||
command: "Runtime.evaluate",
|
||||
params: {
|
||||
expression: expect.stringContaining("delete window[stateKey]"),
|
||||
returnByValue: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("leaves JavaScript dialogs alone when no capture is active", async () => {
|
||||
const contents = new FakeWebContents(28);
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
await expect(tab.captureDialogs?.(async () => "done")).resolves.toEqual({
|
||||
result: "done",
|
||||
dialogs: [],
|
||||
});
|
||||
contents.debugger.emitMessage("Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "Unsaved changes?",
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).not.toContainEqual({
|
||||
command: "Page.handleJavaScriptDialog",
|
||||
params: { accept: false },
|
||||
});
|
||||
});
|
||||
|
||||
test("treats prompt shim drain failures after navigation as no dialogs", async () => {
|
||||
const contents = new FakeWebContents(29);
|
||||
contents.debugger.failPromptDrain = true;
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
await expect(tab.captureDialogs?.(async () => "navigated")).resolves.toEqual({
|
||||
result: "navigated",
|
||||
dialogs: [],
|
||||
});
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Page.enable", params: {} },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: {
|
||||
expression: expect.stringContaining("delete window[stateKey]"),
|
||||
returnByValue: true,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("runs the command without dialog capture when CDP setup fails", async () => {
|
||||
const contents = new FakeWebContents(30);
|
||||
contents.debugger.failedCommandNames.add("Page.enable");
|
||||
const tab = adaptWebContents(contents);
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
|
||||
await expect(tab.captureDialogs?.(async () => "done")).resolves.toEqual({
|
||||
result: "done",
|
||||
dialogs: [],
|
||||
});
|
||||
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
"[browser-automation] Dialog capture unavailable; running command without it",
|
||||
{ contentsId: 30, error: expect.any(Error) },
|
||||
);
|
||||
warn.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
async function flushMicrotasks(): Promise<void> {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T | PromiseLike<T>) => void;
|
||||
const promise = new Promise<T>((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
@@ -1,8 +1,20 @@
|
||||
import type { Rectangle } from "electron";
|
||||
import { ipcMain } from "electron";
|
||||
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type {
|
||||
BrowserAutomationConsoleLogEntry,
|
||||
BrowserAutomationDialogEvent,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { TabContents, BrowserRegistry, TabImage } from "./service.js";
|
||||
import { CdpSessionQueue } from "./cdp-session-queue.js";
|
||||
import {
|
||||
dialogAcceptValue,
|
||||
handledDialogEvent,
|
||||
MAX_DIALOGS_PER_COMMAND,
|
||||
promptShimDrainScript,
|
||||
promptShimInstallScript,
|
||||
promptShimRestoreScript,
|
||||
} from "./dialog-handling.js";
|
||||
import { executeAutomationCommand } from "./service.js";
|
||||
import {
|
||||
listRegisteredPaseoBrowserIds,
|
||||
@@ -14,6 +26,8 @@ import {
|
||||
|
||||
const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
|
||||
const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
|
||||
const cdpQueuesByContentsId = new Map<number, CdpSessionQueue>();
|
||||
const dialogMonitorsByContentsId = new Map<number, DialogMonitor>();
|
||||
const observedContentsIds = new Set<number>();
|
||||
|
||||
interface IpcHandlerRegistry {
|
||||
@@ -24,6 +38,10 @@ interface WebContentsDebugger {
|
||||
isAttached(): boolean;
|
||||
attach(protocolVersion?: string): void;
|
||||
sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown>;
|
||||
on?(
|
||||
event: "message",
|
||||
listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
|
||||
): void;
|
||||
}
|
||||
|
||||
interface ConsoleMessageEmitter {
|
||||
@@ -60,6 +78,8 @@ interface BrowserAutomationWebContents extends ConsoleMessageEmitter {
|
||||
|
||||
export function adaptWebContents(contents: BrowserAutomationWebContents): TabContents {
|
||||
observeConsoleMessages(contents);
|
||||
const cdpQueue = getCdpQueue(contents.id);
|
||||
const dialogMonitor = getDialogMonitor(contents, cdpQueue);
|
||||
return {
|
||||
id: contents.id,
|
||||
getURL: () => contents.getURL(),
|
||||
@@ -76,15 +96,27 @@ export function adaptWebContents(contents: BrowserAutomationWebContents): TabCon
|
||||
capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions),
|
||||
invalidate: () => contents.invalidate(),
|
||||
getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
|
||||
sendDebugCommand: async (command: string, params?: Record<string, unknown>) => {
|
||||
if (!contents.debugger.isAttached()) {
|
||||
contents.debugger.attach("1.3");
|
||||
}
|
||||
return contents.debugger.sendCommand(command, params ?? {});
|
||||
},
|
||||
captureDialogs: (task) => dialogMonitor.capture(task),
|
||||
sendDebugCommand: (command: string, params?: Record<string, unknown>) =>
|
||||
cdpQueue.run(async () => {
|
||||
if (!contents.debugger.isAttached()) {
|
||||
contents.debugger.attach("1.3");
|
||||
}
|
||||
return contents.debugger.sendCommand(command, params ?? {});
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function getCdpQueue(contentsId: number): CdpSessionQueue {
|
||||
const existing = cdpQueuesByContentsId.get(contentsId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const queue = new CdpSessionQueue();
|
||||
cdpQueuesByContentsId.set(contentsId, queue);
|
||||
return queue;
|
||||
}
|
||||
|
||||
function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
|
||||
if (observedContentsIds.has(contents.id)) {
|
||||
return;
|
||||
@@ -99,6 +131,192 @@ function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
|
||||
contents.once("destroyed", () => {
|
||||
observedContentsIds.delete(contents.id);
|
||||
consoleMessagesByContentsId.delete(contents.id);
|
||||
cdpQueuesByContentsId.delete(contents.id);
|
||||
dialogMonitorsByContentsId.delete(contents.id);
|
||||
});
|
||||
}
|
||||
|
||||
function getDialogMonitor(
|
||||
contents: BrowserAutomationWebContents,
|
||||
cdpQueue: CdpSessionQueue,
|
||||
): DialogMonitor {
|
||||
const existing = dialogMonitorsByContentsId.get(contents.id);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const monitor = new DialogMonitor(contents, cdpQueue);
|
||||
dialogMonitorsByContentsId.set(contents.id, monitor);
|
||||
return monitor;
|
||||
}
|
||||
|
||||
class DialogMonitor {
|
||||
private enabled = false;
|
||||
private listenerRegistered = false;
|
||||
private readonly activeCollectors: DialogCollector[] = [];
|
||||
|
||||
public constructor(
|
||||
private readonly contents: BrowserAutomationWebContents,
|
||||
private readonly cdpQueue: CdpSessionQueue,
|
||||
) {}
|
||||
|
||||
public async capture<T>(
|
||||
task: () => Promise<T>,
|
||||
): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }> {
|
||||
const collector: DialogCollector = { dialogs: [] };
|
||||
try {
|
||||
await this.enable();
|
||||
await this.installPromptShim();
|
||||
} catch (error) {
|
||||
console.warn("[browser-automation] Dialog capture unavailable; running command without it", {
|
||||
contentsId: this.contents.id,
|
||||
error,
|
||||
});
|
||||
return { result: await task(), dialogs: [] };
|
||||
}
|
||||
this.activeCollectors.push(collector);
|
||||
try {
|
||||
const result = await task();
|
||||
this.recordPromptShimDialogs(await this.drainPromptShim());
|
||||
return { result, dialogs: collector.dialogs };
|
||||
} finally {
|
||||
const index = this.activeCollectors.indexOf(collector);
|
||||
if (index >= 0) {
|
||||
this.activeCollectors.splice(index, 1);
|
||||
}
|
||||
if (this.activeCollectors.length === 0) {
|
||||
await this.restorePromptShim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async enable(): Promise<void> {
|
||||
if (this.enabled) {
|
||||
return;
|
||||
}
|
||||
if (!this.contents.debugger.on) {
|
||||
return;
|
||||
}
|
||||
if (!this.listenerRegistered) {
|
||||
this.listenerRegistered = true;
|
||||
this.contents.debugger.on("message", (_event, method, params) => {
|
||||
if (method !== "Page.javascriptDialogOpening") {
|
||||
return;
|
||||
}
|
||||
if (this.activeCollectors.length === 0) {
|
||||
return;
|
||||
}
|
||||
void this.handleOpening(params ?? {});
|
||||
});
|
||||
}
|
||||
await this.sendDebugCommand("Page.enable");
|
||||
this.enabled = true;
|
||||
}
|
||||
|
||||
private async handleOpening(params: Record<string, unknown>): Promise<void> {
|
||||
const event = handledDialogEvent(params);
|
||||
for (const collector of this.activeCollectors) {
|
||||
this.recordDialogs(collector, [event]);
|
||||
}
|
||||
await this.sendDialogResponseCommand("Page.handleJavaScriptDialog", {
|
||||
accept: dialogAcceptValue(event.type),
|
||||
});
|
||||
}
|
||||
|
||||
private async installPromptShim(): Promise<void> {
|
||||
await this.sendDebugCommand("Runtime.evaluate", {
|
||||
expression: promptShimInstallScript(),
|
||||
returnByValue: true,
|
||||
});
|
||||
}
|
||||
|
||||
private async drainPromptShim(): Promise<BrowserAutomationDialogEvent[]> {
|
||||
try {
|
||||
const result = (await this.sendDebugCommand("Runtime.evaluate", {
|
||||
expression: promptShimDrainScript(),
|
||||
returnByValue: true,
|
||||
})) as { result?: { value?: unknown } };
|
||||
return parsePromptShimDialogs(result.result?.value);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private async restorePromptShim(): Promise<void> {
|
||||
try {
|
||||
await this.sendDebugCommand("Runtime.evaluate", {
|
||||
expression: promptShimRestoreScript(),
|
||||
returnByValue: true,
|
||||
});
|
||||
} catch {
|
||||
// Navigation can destroy the execution context before cleanup runs; the next page has no shim.
|
||||
}
|
||||
}
|
||||
|
||||
private recordDialogs(collector: DialogCollector, dialogs: BrowserAutomationDialogEvent[]): void {
|
||||
for (const dialog of dialogs) {
|
||||
if (collector.dialogs.length >= MAX_DIALOGS_PER_COMMAND) {
|
||||
return;
|
||||
}
|
||||
collector.dialogs.push(dialog);
|
||||
}
|
||||
}
|
||||
|
||||
private recordPromptShimDialogs(dialogs: BrowserAutomationDialogEvent[]): void {
|
||||
for (const collector of this.activeCollectors) {
|
||||
this.recordDialogs(collector, dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
private async sendDebugCommand(
|
||||
command: string,
|
||||
params?: Record<string, unknown>,
|
||||
): Promise<unknown> {
|
||||
return this.cdpQueue.run(async () => {
|
||||
if (!this.contents.debugger.isAttached()) {
|
||||
this.contents.debugger.attach("1.3");
|
||||
}
|
||||
return this.contents.debugger.sendCommand(command, params ?? {});
|
||||
});
|
||||
}
|
||||
|
||||
private async sendDialogResponseCommand(
|
||||
command: string,
|
||||
params?: Record<string, unknown>,
|
||||
): Promise<unknown> {
|
||||
// Dialogs can block the CDP command that opened them, so the unblocker must not wait behind
|
||||
// the per-tab command queue.
|
||||
if (!this.contents.debugger.isAttached()) {
|
||||
this.contents.debugger.attach("1.3");
|
||||
}
|
||||
return this.contents.debugger.sendCommand(command, params ?? {});
|
||||
}
|
||||
}
|
||||
|
||||
interface DialogCollector {
|
||||
dialogs: BrowserAutomationDialogEvent[];
|
||||
}
|
||||
|
||||
function parsePromptShimDialogs(value: unknown): BrowserAutomationDialogEvent[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.flatMap((entry): BrowserAutomationDialogEvent[] => {
|
||||
if (!entry || typeof entry !== "object") {
|
||||
return [];
|
||||
}
|
||||
const record = entry as Record<string, unknown>;
|
||||
if (record.type !== "prompt" || record.action !== "dismissed") {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
type: "prompt",
|
||||
message: typeof record.message === "string" ? record.message : "",
|
||||
...(typeof record.defaultValue === "string" ? { defaultValue: record.defaultValue } : {}),
|
||||
action: "dismissed",
|
||||
timestamp: typeof record.timestamp === "number" ? record.timestamp : Date.now(),
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -3,79 +3,162 @@ import { BrowserSnapshotEngine, type SnapshotPage } from "./snapshot-engine.js";
|
||||
|
||||
class SnapshotFixture implements SnapshotPage {
|
||||
public currentUrl = "https://example.com/form";
|
||||
public actionResult: unknown = true;
|
||||
public actionResult: unknown = { ok: true };
|
||||
public alreadyTruncated = false;
|
||||
public snapshotNodes: unknown[] = [
|
||||
{
|
||||
kind: "role",
|
||||
role: "heading",
|
||||
name: "Settings",
|
||||
tagName: "h1",
|
||||
attributes: ["level=1"],
|
||||
children: [],
|
||||
},
|
||||
{ kind: "text", text: "Connected as Maya" },
|
||||
{
|
||||
kind: "role",
|
||||
role: "button",
|
||||
name: "Save changes",
|
||||
tagName: "button",
|
||||
attributes: [],
|
||||
ref: "@e1",
|
||||
fingerprint: {
|
||||
role: "button",
|
||||
name: "Save changes",
|
||||
tagName: "button",
|
||||
type: "",
|
||||
ariaLabel: "",
|
||||
},
|
||||
children: [],
|
||||
},
|
||||
];
|
||||
|
||||
public getURL(): string {
|
||||
return this.currentUrl;
|
||||
}
|
||||
|
||||
public async executeJavaScript(code: string): Promise<unknown> {
|
||||
if (code.includes("CANDIDATE_SELECTOR")) {
|
||||
return JSON.stringify([
|
||||
{
|
||||
role: "textbox",
|
||||
tagName: "input",
|
||||
text: "Name",
|
||||
selector: "#name",
|
||||
attributes: { id: "name", type: "text" },
|
||||
if (code.includes("__PASEO_ARIA_SNAPSHOT__")) {
|
||||
return JSON.stringify({
|
||||
marker: "__PASEO_ARIA_SNAPSHOT__",
|
||||
root: {
|
||||
kind: "role",
|
||||
role: "document",
|
||||
name: "Fixture",
|
||||
tagName: "document",
|
||||
attributes: [],
|
||||
children: this.snapshotNodes,
|
||||
},
|
||||
{
|
||||
role: "button",
|
||||
tagName: "button",
|
||||
text: "Drop",
|
||||
selector: "#drop",
|
||||
attributes: { id: "drop" },
|
||||
},
|
||||
]);
|
||||
refs: [
|
||||
{
|
||||
ref: "@e1",
|
||||
fingerprint: {
|
||||
role: "button",
|
||||
name: "Save changes",
|
||||
tagName: "button",
|
||||
type: "",
|
||||
ariaLabel: "",
|
||||
},
|
||||
},
|
||||
],
|
||||
truncated: this.alreadyTruncated,
|
||||
stats: { nodeCount: 4, refCount: 1, textLength: 0, iframeCount: 0, maxDepth: 1 },
|
||||
});
|
||||
}
|
||||
return this.actionResult;
|
||||
}
|
||||
}
|
||||
|
||||
describe("BrowserSnapshotEngine", () => {
|
||||
it("treats a false result from a ref action script as a stale ref", async () => {
|
||||
it("renders a hierarchical ARIA YAML snapshot with static text and actionable refs", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
page.actionResult = false;
|
||||
|
||||
await expect(engine.click({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: "stale_ref",
|
||||
});
|
||||
await expect(
|
||||
engine.select({ browserId: "browser-1", page, ref: "@e1", value: "us" }),
|
||||
).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: "stale_ref",
|
||||
await expect(engine.snapshot({ browserId: "browser-1", page })).resolves.toEqual({
|
||||
format: "aria-yaml",
|
||||
snapshot: [
|
||||
'- document "Fixture"',
|
||||
' - heading "Settings" [level=1]',
|
||||
' - text: "Connected as Maya"',
|
||||
' - button "Save changes" [ref=@e1]',
|
||||
].join("\n"),
|
||||
truncated: false,
|
||||
stats: { nodeCount: 4, refCount: 1, textLength: 119, iframeCount: 0, maxDepth: 1 },
|
||||
});
|
||||
});
|
||||
|
||||
it("treats a false result from optional ref text/key actions as a stale ref", async () => {
|
||||
it("builds a runtime ref expression with the snapshot fingerprint", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
page.actionResult = false;
|
||||
page.currentUrl = "https://example.com/form?panel=advanced";
|
||||
|
||||
await expect(
|
||||
engine.typeText({ browserId: "browser-1", page, ref: "@e1", text: "Ada" }),
|
||||
).resolves.toEqual({ ok: false, reason: "stale_ref" });
|
||||
await expect(
|
||||
engine.keypress({ browserId: "browser-1", page, ref: "@e1", key: "Enter" }),
|
||||
).resolves.toEqual({ ok: false, reason: "stale_ref" });
|
||||
expect(engine.runtimeElementExpression({ browserId: "browser-1", ref: "@e1" })).toContain(
|
||||
'"name":"Save changes"',
|
||||
);
|
||||
});
|
||||
|
||||
it("treats a false result from drag as a stale ref", async () => {
|
||||
it("treats missing host-side ref metadata as a stale ref", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
page.actionResult = false;
|
||||
expect(engine.runtimeElementExpression({ browserId: "browser-1", ref: "@e2" })).toEqual({
|
||||
ok: false,
|
||||
reason: "missing_ref",
|
||||
});
|
||||
});
|
||||
|
||||
await expect(
|
||||
engine.drag({ browserId: "browser-1", page, sourceRef: "@e1", targetRef: "@e2" }),
|
||||
).resolves.toEqual({ ok: false, reason: "stale_ref" });
|
||||
it("marks rendered output truncation explicitly and deterministically", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
page.snapshotNodes = [
|
||||
{
|
||||
kind: "text",
|
||||
text: "A".repeat(81_000),
|
||||
},
|
||||
];
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
|
||||
const snapshot = await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
expect(snapshot.truncated).toBe(true);
|
||||
expect(snapshot.snapshot.endsWith('- text: "Snapshot truncated."')).toBe(true);
|
||||
expect(snapshot.stats.textLength).toBeLessThanOrEqual(80_000);
|
||||
});
|
||||
|
||||
it("keeps the last rendered node when a short snapshot was capped by node count", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
page.alreadyTruncated = true;
|
||||
page.snapshotNodes = [
|
||||
{
|
||||
kind: "role",
|
||||
role: "button",
|
||||
name: "Final action",
|
||||
tagName: "button",
|
||||
attributes: [],
|
||||
ref: "@e1",
|
||||
fingerprint: {
|
||||
role: "button",
|
||||
name: "Final action",
|
||||
tagName: "button",
|
||||
type: "",
|
||||
ariaLabel: "",
|
||||
},
|
||||
children: [],
|
||||
},
|
||||
];
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
|
||||
const snapshot = await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
expect(snapshot.truncated).toBe(true);
|
||||
expect(snapshot.snapshot).toBe(
|
||||
[
|
||||
'- document "Fixture"',
|
||||
' - button "Final action" [ref=@e1]',
|
||||
'- text: "Snapshot truncated."',
|
||||
].join("\n"),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,24 +1,60 @@
|
||||
import { ARIA_SNAPSHOT_SCRIPT, ARIA_SNAPSHOT_SCRIPT_MARKER } from "./aria-snapshot-script.js";
|
||||
|
||||
export interface SnapshotPage {
|
||||
getURL(): string;
|
||||
executeJavaScript(code: string): Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface BrowserSnapshotElement extends RawSnapshotElement {
|
||||
ref: string;
|
||||
export interface BrowserAriaSnapshot {
|
||||
format: "aria-yaml";
|
||||
snapshot: string;
|
||||
truncated: boolean;
|
||||
stats: BrowserAriaSnapshotStats;
|
||||
}
|
||||
|
||||
interface RawSnapshotElement {
|
||||
export interface BrowserAriaSnapshotStats {
|
||||
nodeCount: number;
|
||||
refCount: number;
|
||||
textLength: number;
|
||||
iframeCount?: number;
|
||||
maxDepth?: number;
|
||||
}
|
||||
|
||||
interface SnapshotNode {
|
||||
kind: "role" | "text" | "group";
|
||||
role?: string;
|
||||
name?: string;
|
||||
text?: string;
|
||||
tagName?: string;
|
||||
attributes?: string[];
|
||||
ref?: string;
|
||||
fingerprint?: BrowserRefFingerprint;
|
||||
children?: SnapshotNode[];
|
||||
}
|
||||
|
||||
interface BrowserRefFingerprint {
|
||||
role: string;
|
||||
name: string;
|
||||
tagName: string;
|
||||
text: string;
|
||||
selector: string;
|
||||
attributes: Record<string, string>;
|
||||
type: string;
|
||||
ariaLabel: string;
|
||||
}
|
||||
|
||||
interface RawAriaSnapshot {
|
||||
marker: string;
|
||||
root: SnapshotNode;
|
||||
refs: BrowserRefMetadata[];
|
||||
truncated: boolean;
|
||||
stats: BrowserAriaSnapshotStats;
|
||||
}
|
||||
|
||||
interface BrowserRefMetadata {
|
||||
ref: string;
|
||||
fingerprint: BrowserRefFingerprint;
|
||||
}
|
||||
|
||||
interface BrowserRefState {
|
||||
nextRefNumber: number;
|
||||
url: string;
|
||||
refs: Map<string, RawSnapshotElement>;
|
||||
refs: Map<string, BrowserRefMetadata>;
|
||||
}
|
||||
|
||||
export type BrowserRefActionResult =
|
||||
@@ -26,45 +62,31 @@ export type BrowserRefActionResult =
|
||||
| { ok: false; reason: "stale_ref" | "missing_ref" };
|
||||
type BrowserRefFailure = Extract<BrowserRefActionResult, { ok: false }>;
|
||||
|
||||
type BrowserRefResolveResult = { ok: true; element: RawSnapshotElement } | BrowserRefFailure;
|
||||
type BrowserRefResolveResult = { ok: true; metadata: BrowserRefMetadata } | BrowserRefFailure;
|
||||
|
||||
const TRUNCATION_MARKER = '- text: "Snapshot truncated."';
|
||||
const MAX_RENDERED_TEXT_LENGTH = 80_000;
|
||||
|
||||
export class BrowserSnapshotEngine {
|
||||
private readonly statesByBrowserId = new Map<string, BrowserRefState>();
|
||||
|
||||
async snapshot(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
}): Promise<BrowserSnapshotElement[]> {
|
||||
const rawElements = parseRawSnapshotElements(
|
||||
await input.page.executeJavaScript(SNAPSHOT_SCRIPT),
|
||||
);
|
||||
const state = {
|
||||
nextRefNumber: 1,
|
||||
url: input.page.getURL(),
|
||||
refs: new Map<string, RawSnapshotElement>(),
|
||||
};
|
||||
const elements = rawElements.map((element) => {
|
||||
const ref = `@e${state.nextRefNumber++}`;
|
||||
state.refs.set(ref, element);
|
||||
return {
|
||||
ref,
|
||||
role: element.role,
|
||||
tagName: element.tagName,
|
||||
text: element.text,
|
||||
selector: element.selector,
|
||||
attributes: element.attributes,
|
||||
};
|
||||
async snapshot(input: { browserId: string; page: SnapshotPage }): Promise<BrowserAriaSnapshot> {
|
||||
const rawSnapshot = parseAriaSnapshot(await input.page.executeJavaScript(ARIA_SNAPSHOT_SCRIPT));
|
||||
const rendered = renderSnapshot(rawSnapshot.root);
|
||||
const capped = capRenderedSnapshot(rendered, rawSnapshot.truncated);
|
||||
this.statesByBrowserId.set(input.browserId, {
|
||||
refs: new Map(rawSnapshot.refs.map((ref) => [ref.ref, ref])),
|
||||
});
|
||||
this.statesByBrowserId.set(input.browserId, state);
|
||||
return elements;
|
||||
}
|
||||
|
||||
async click(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildClickScript(selector));
|
||||
return {
|
||||
format: "aria-yaml",
|
||||
snapshot: capped.snapshot,
|
||||
truncated: capped.truncated,
|
||||
stats: {
|
||||
...rawSnapshot.stats,
|
||||
refCount: rawSnapshot.refs.length,
|
||||
textLength: capped.snapshot.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async fill(input: {
|
||||
@@ -73,39 +95,7 @@ export class BrowserSnapshotEngine {
|
||||
ref: string;
|
||||
value: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildFillScript(selector, input.value));
|
||||
}
|
||||
|
||||
async typeText(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref?: string;
|
||||
text: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
const selector = this.resolveOptionalRef(input);
|
||||
if (!selector.ok) {
|
||||
return selector;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildTypeScript(selector.selector, input.text),
|
||||
);
|
||||
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
}
|
||||
|
||||
async keypress(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref?: string;
|
||||
key: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
const selector = this.resolveOptionalRef(input);
|
||||
if (!selector.ok) {
|
||||
return selector;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildKeypressScript(selector.selector, input.key),
|
||||
);
|
||||
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
return this.runRefScript(input, (ref) => buildFillScript(ref, input.value));
|
||||
}
|
||||
|
||||
async select(input: {
|
||||
@@ -114,123 +104,214 @@ export class BrowserSnapshotEngine {
|
||||
ref: string;
|
||||
value: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildSelectScript(selector, input.value));
|
||||
}
|
||||
|
||||
async hover(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildHoverScript(selector));
|
||||
}
|
||||
|
||||
async drag(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
sourceRef: string;
|
||||
targetRef: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
const source = this.resolveRef({
|
||||
browserId: input.browserId,
|
||||
page: input.page,
|
||||
ref: input.sourceRef,
|
||||
});
|
||||
if (!source.ok) {
|
||||
return source;
|
||||
}
|
||||
const target = this.resolveRef({
|
||||
browserId: input.browserId,
|
||||
page: input.page,
|
||||
ref: input.targetRef,
|
||||
});
|
||||
if (!target.ok) {
|
||||
return target;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildDragScript(source.element.selector, target.element.selector),
|
||||
);
|
||||
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
return this.runRefScript(input, (ref) => buildSelectScript(ref, input.value));
|
||||
}
|
||||
|
||||
clearBrowser(browserId: string): void {
|
||||
this.statesByBrowserId.delete(browserId);
|
||||
}
|
||||
|
||||
selectorForRef(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): { ok: true; selector: string } | BrowserRefFailure {
|
||||
runtimeElementExpression(input: { browserId: string; ref: string }): string | BrowserRefFailure {
|
||||
const resolved = this.resolveRef(input);
|
||||
if (!resolved.ok) {
|
||||
return resolved;
|
||||
}
|
||||
return { ok: true, selector: resolved.element.selector };
|
||||
return buildRuntimeElementExpression(resolved.metadata);
|
||||
}
|
||||
|
||||
private async runRefScript(
|
||||
input: { browserId: string; page: SnapshotPage; ref: string },
|
||||
buildScript: (selector: string) => string,
|
||||
buildScript: (ref: BrowserRefMetadata) => string,
|
||||
): Promise<BrowserRefActionResult> {
|
||||
const resolved = this.resolveRef(input);
|
||||
if (!resolved.ok) {
|
||||
return resolved;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(buildScript(resolved.element.selector));
|
||||
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
const result = await input.page.executeJavaScript(buildScript(resolved.metadata));
|
||||
return readActionResult(result, true);
|
||||
}
|
||||
|
||||
private resolveRef(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): BrowserRefResolveResult {
|
||||
private resolveRef(input: { browserId: string; ref: string }): BrowserRefResolveResult {
|
||||
const state = this.statesByBrowserId.get(input.browserId);
|
||||
if (!state || state.url !== input.page.getURL()) {
|
||||
if (!state) {
|
||||
return { ok: false, reason: "stale_ref" };
|
||||
}
|
||||
const element = state.refs.get(input.ref);
|
||||
if (!element) {
|
||||
const metadata = state.refs.get(input.ref);
|
||||
if (!metadata) {
|
||||
return { ok: false, reason: "missing_ref" };
|
||||
}
|
||||
return { ok: true, element };
|
||||
}
|
||||
|
||||
private resolveOptionalRef(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref?: string;
|
||||
}): { ok: true; selector: string | undefined } | BrowserRefFailure {
|
||||
if (!input.ref) {
|
||||
return { ok: true, selector: undefined };
|
||||
}
|
||||
const resolved = this.resolveRef({
|
||||
browserId: input.browserId,
|
||||
page: input.page,
|
||||
ref: input.ref,
|
||||
});
|
||||
if (!resolved.ok) {
|
||||
return resolved;
|
||||
}
|
||||
return { ok: true, selector: resolved.element.selector };
|
||||
return { ok: true, metadata };
|
||||
}
|
||||
}
|
||||
|
||||
function buildClickScript(selector: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
element.click();
|
||||
return true;
|
||||
})()`;
|
||||
function readActionResult(value: unknown, staleWhenFalse: boolean): BrowserRefActionResult {
|
||||
if (value === false && staleWhenFalse) {
|
||||
return { ok: false, reason: "stale_ref" };
|
||||
}
|
||||
if (isActionFailure(value)) {
|
||||
return { ok: false, reason: value.reason };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
function buildFillScript(selector: string, value: string): string {
|
||||
function isActionFailure(value: unknown): value is BrowserRefFailure {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const reason = (value as Record<string, unknown>).reason;
|
||||
return reason === "stale_ref" || reason === "missing_ref";
|
||||
}
|
||||
|
||||
function parseAriaSnapshot(value: unknown): RawAriaSnapshot {
|
||||
const parsed = typeof value === "string" ? JSON.parse(value) : value;
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
return emptySnapshot();
|
||||
}
|
||||
const record = parsed as Record<string, unknown>;
|
||||
if (record.marker !== ARIA_SNAPSHOT_SCRIPT_MARKER) {
|
||||
return emptySnapshot();
|
||||
}
|
||||
const root = parseSnapshotNode(record.root);
|
||||
return {
|
||||
marker: ARIA_SNAPSHOT_SCRIPT_MARKER,
|
||||
root: root ?? emptySnapshot().root,
|
||||
refs: parseRefs(record.refs),
|
||||
truncated: record.truncated === true,
|
||||
stats: parseStats(record.stats),
|
||||
};
|
||||
}
|
||||
|
||||
function emptySnapshot(): RawAriaSnapshot {
|
||||
return {
|
||||
marker: ARIA_SNAPSHOT_SCRIPT_MARKER,
|
||||
root: { kind: "role", role: "document", name: "", tagName: "document", children: [] },
|
||||
refs: [],
|
||||
truncated: false,
|
||||
stats: { nodeCount: 0, refCount: 0, textLength: 0, iframeCount: 0, maxDepth: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
function parseSnapshotNode(value: unknown): SnapshotNode | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
const kind = record.kind;
|
||||
if (kind !== "role" && kind !== "text" && kind !== "group") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
kind,
|
||||
...(readString(record.role) ? { role: readString(record.role) ?? undefined } : {}),
|
||||
...(readString(record.name) ? { name: readString(record.name) ?? undefined } : {}),
|
||||
...(readString(record.text) ? { text: readString(record.text) ?? undefined } : {}),
|
||||
...(readString(record.tagName) ? { tagName: readString(record.tagName) ?? undefined } : {}),
|
||||
...(readString(record.ref) ? { ref: readString(record.ref) ?? undefined } : {}),
|
||||
...(parseFingerprint(record.fingerprint)
|
||||
? { fingerprint: parseFingerprint(record.fingerprint) ?? undefined }
|
||||
: {}),
|
||||
attributes: readStringArray(record.attributes),
|
||||
children: Array.isArray(record.children)
|
||||
? record.children.flatMap((child): SnapshotNode[] => {
|
||||
const parsed = parseSnapshotNode(child);
|
||||
return parsed ? [parsed] : [];
|
||||
})
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
function parseRefs(value: unknown): BrowserRefMetadata[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.flatMap((item): BrowserRefMetadata[] => {
|
||||
if (!item || typeof item !== "object") {
|
||||
return [];
|
||||
}
|
||||
const record = item as Record<string, unknown>;
|
||||
const ref = readString(record.ref);
|
||||
const fingerprint = parseFingerprint(record.fingerprint);
|
||||
return ref && fingerprint ? [{ ref, fingerprint }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
function parseFingerprint(value: unknown): BrowserRefFingerprint | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
const role = readString(record.role);
|
||||
const name = readString(record.name);
|
||||
const tagName = readString(record.tagName);
|
||||
const type = readString(record.type);
|
||||
const ariaLabel = readString(record.ariaLabel);
|
||||
if (role === null || name === null || tagName === null || type === null || ariaLabel === null) {
|
||||
return null;
|
||||
}
|
||||
return { role, name, tagName, type, ariaLabel };
|
||||
}
|
||||
|
||||
function parseStats(value: unknown): BrowserAriaSnapshotStats {
|
||||
if (!value || typeof value !== "object") {
|
||||
return { nodeCount: 0, refCount: 0, textLength: 0 };
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return {
|
||||
nodeCount: readNumber(record.nodeCount) ?? 0,
|
||||
refCount: readNumber(record.refCount) ?? 0,
|
||||
textLength: readNumber(record.textLength) ?? 0,
|
||||
...(readNumber(record.iframeCount) !== null
|
||||
? { iframeCount: readNumber(record.iframeCount) ?? undefined }
|
||||
: {}),
|
||||
...(readNumber(record.maxDepth) !== null
|
||||
? { maxDepth: readNumber(record.maxDepth) ?? undefined }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function renderSnapshot(root: SnapshotNode): string {
|
||||
const lines = renderNode(root, 0);
|
||||
return lines.length > 0 ? lines.join("\n") : "- document";
|
||||
}
|
||||
|
||||
function renderNode(node: SnapshotNode, depth: number): string[] {
|
||||
if (node.kind === "group") {
|
||||
return (node.children ?? []).flatMap((child) => renderNode(child, depth));
|
||||
}
|
||||
const indent = " ".repeat(depth);
|
||||
if (node.kind === "text") {
|
||||
return [`${indent}- text: ${JSON.stringify(node.text ?? "")}`];
|
||||
}
|
||||
const attrs = [...(node.attributes ?? [])];
|
||||
if (node.ref) {
|
||||
attrs.push(`ref=${node.ref}`);
|
||||
}
|
||||
const suffix = attrs.length > 0 ? ` [${attrs.join(" ")}]` : "";
|
||||
const ownLine = `${indent}- ${node.role ?? "generic"}${node.name ? ` ${JSON.stringify(node.name)}` : ""}${suffix}`;
|
||||
const childLines = (node.children ?? []).flatMap((child) => renderNode(child, depth + 1));
|
||||
return [ownLine, ...childLines];
|
||||
}
|
||||
|
||||
function capRenderedSnapshot(
|
||||
snapshot: string,
|
||||
alreadyTruncated: boolean,
|
||||
): { snapshot: string; truncated: boolean } {
|
||||
if (snapshot.length <= MAX_RENDERED_TEXT_LENGTH && !alreadyTruncated) {
|
||||
return { snapshot, truncated: false };
|
||||
}
|
||||
if (snapshot.length + 1 + TRUNCATION_MARKER.length <= MAX_RENDERED_TEXT_LENGTH) {
|
||||
return { snapshot: `${snapshot}\n${TRUNCATION_MARKER}`, truncated: true };
|
||||
}
|
||||
const availableLength = MAX_RENDERED_TEXT_LENGTH - TRUNCATION_MARKER.length - 1;
|
||||
const capped = snapshot.slice(0, Math.max(0, availableLength)).replace(/\n[^\n]*$/, "");
|
||||
return { snapshot: `${capped}\n${TRUNCATION_MARKER}`, truncated: true };
|
||||
}
|
||||
|
||||
function buildFillScript(metadata: BrowserRefMetadata, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
const resolved = ${buildResolveExpression(metadata)};
|
||||
if (!resolved.ok) return resolved;
|
||||
const element = resolved.element;
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
element.focus();
|
||||
const nextValue = ${JSON.stringify(value)};
|
||||
@@ -238,51 +319,19 @@ function buildFillScript(selector: string, value: string): string {
|
||||
element.value = nextValue;
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
return { ok: true };
|
||||
}
|
||||
element.textContent = nextValue;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: nextValue }));
|
||||
return true;
|
||||
return { ok: true };
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildTypeScript(selector: string | undefined, text: string): string {
|
||||
function buildSelectScript(metadata: BrowserRefMetadata, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
|
||||
if (!element) return false;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
element.focus?.();
|
||||
const text = ${JSON.stringify(text)};
|
||||
if ('value' in element) {
|
||||
element.value = String(element.value || '') + text;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
}
|
||||
element.textContent = String(element.textContent || '') + text;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildKeypressScript(selector: string | undefined, key: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
|
||||
if (!element) return false;
|
||||
element.focus?.();
|
||||
const key = ${JSON.stringify(key)};
|
||||
const eventInit = { bubbles: true, cancelable: true, key };
|
||||
element.dispatchEvent(new KeyboardEvent('keydown', eventInit));
|
||||
element.dispatchEvent(new KeyboardEvent('keypress', eventInit));
|
||||
element.dispatchEvent(new KeyboardEvent('keyup', eventInit));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildSelectScript(selector: string, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
const resolved = ${buildResolveExpression(metadata)};
|
||||
if (!resolved.ok) return resolved;
|
||||
const element = resolved.element;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
element.focus?.();
|
||||
const nextValue = ${JSON.stringify(value)};
|
||||
@@ -290,205 +339,34 @@ function buildSelectScript(selector: string, value: string): string {
|
||||
element.value = nextValue;
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
return { ok: true };
|
||||
}
|
||||
return false;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildHoverScript(selector: string): string {
|
||||
function buildRuntimeElementExpression(metadata: BrowserRefMetadata): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
const rect = element.getBoundingClientRect();
|
||||
const eventInit = {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
clientX: rect.left + rect.width / 2,
|
||||
clientY: rect.top + rect.height / 2,
|
||||
screenX: window.screenX + rect.left + rect.width / 2,
|
||||
screenY: window.screenY + rect.top + rect.height / 2,
|
||||
view: window,
|
||||
};
|
||||
element.dispatchEvent(new MouseEvent('mouseover', eventInit));
|
||||
element.dispatchEvent(new MouseEvent('mouseenter', eventInit));
|
||||
element.dispatchEvent(new MouseEvent('mousemove', eventInit));
|
||||
return true;
|
||||
const resolved = ${buildResolveExpression(metadata)};
|
||||
if (!resolved.ok) return null;
|
||||
return resolved.element;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildDragScript(sourceSelector: string, targetSelector: string): string {
|
||||
return String.raw`(() => {
|
||||
const source = document.querySelector(${JSON.stringify(sourceSelector)});
|
||||
const target = document.querySelector(${JSON.stringify(targetSelector)});
|
||||
if (!source || !target) return false;
|
||||
source.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
target.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
const data = new DataTransfer();
|
||||
const sourceRect = source.getBoundingClientRect();
|
||||
const targetRect = target.getBoundingClientRect();
|
||||
function eventInit(rect) {
|
||||
return {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
clientX: rect.left + rect.width / 2,
|
||||
clientY: rect.top + rect.height / 2,
|
||||
screenX: window.screenX + rect.left + rect.width / 2,
|
||||
screenY: window.screenY + rect.top + rect.height / 2,
|
||||
dataTransfer: data,
|
||||
view: window,
|
||||
};
|
||||
}
|
||||
source.dispatchEvent(new MouseEvent('mousedown', eventInit(sourceRect)));
|
||||
source.dispatchEvent(new DragEvent('dragstart', eventInit(sourceRect)));
|
||||
target.dispatchEvent(new DragEvent('dragenter', eventInit(targetRect)));
|
||||
target.dispatchEvent(new DragEvent('dragover', eventInit(targetRect)));
|
||||
target.dispatchEvent(new DragEvent('drop', eventInit(targetRect)));
|
||||
source.dispatchEvent(new DragEvent('dragend', eventInit(sourceRect)));
|
||||
target.dispatchEvent(new MouseEvent('mouseup', eventInit(targetRect)));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function parseRawSnapshotElements(value: unknown): RawSnapshotElement[] {
|
||||
const parsed = typeof value === "string" ? JSON.parse(value) : value;
|
||||
if (!Array.isArray(parsed)) {
|
||||
return [];
|
||||
}
|
||||
return parsed.flatMap((item): RawSnapshotElement[] => {
|
||||
if (!item || typeof item !== "object") {
|
||||
return [];
|
||||
}
|
||||
const record = item as Record<string, unknown>;
|
||||
const selector = readString(record.selector);
|
||||
if (!selector) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
role: readString(record.role) || "generic",
|
||||
tagName: (readString(record.tagName) || "element").toLowerCase(),
|
||||
text: readString(record.text) || "",
|
||||
selector,
|
||||
attributes: readAttributes(record.attributes),
|
||||
},
|
||||
];
|
||||
});
|
||||
function buildResolveExpression(metadata: BrowserRefMetadata): string {
|
||||
return `window.__PASEO_BROWSER_AUTOMATION__?.resolve(${JSON.stringify(metadata.ref)}, ${JSON.stringify(metadata.fingerprint)}) ?? { ok: false, reason: 'stale_ref' }`;
|
||||
}
|
||||
|
||||
function readString(value: unknown): string | null {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
function readAttributes(value: unknown): Record<string, string> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return {};
|
||||
}
|
||||
const result: Record<string, string> = {};
|
||||
for (const [key, attributeValue] of Object.entries(value)) {
|
||||
if (typeof attributeValue === "string") {
|
||||
result[key] = attributeValue;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
function readStringArray(value: unknown): string[] {
|
||||
return Array.isArray(value)
|
||||
? value.filter((item): item is string => typeof item === "string")
|
||||
: [];
|
||||
}
|
||||
|
||||
const SNAPSHOT_SCRIPT = String.raw`(() => {
|
||||
const MAX_ELEMENTS = 200;
|
||||
const CANDIDATE_SELECTOR = [
|
||||
'a[href]',
|
||||
'button',
|
||||
'input',
|
||||
'textarea',
|
||||
'select',
|
||||
'summary',
|
||||
'[role]',
|
||||
'[tabindex]:not([tabindex="-1"])',
|
||||
'[contenteditable=""]',
|
||||
'[contenteditable="true"]'
|
||||
].join(',');
|
||||
|
||||
function cssEscape(value) {
|
||||
if (window.CSS && typeof window.CSS.escape === 'function') {
|
||||
return window.CSS.escape(value);
|
||||
}
|
||||
return String(value).replace(/[^a-zA-Z0-9_-]/g, '\\$&');
|
||||
}
|
||||
|
||||
function isVisible(element) {
|
||||
const style = window.getComputedStyle(element);
|
||||
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
|
||||
return false;
|
||||
}
|
||||
const rect = element.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}
|
||||
|
||||
function roleFor(element) {
|
||||
const explicit = element.getAttribute('role');
|
||||
if (explicit) return explicit;
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (tag === 'a') return 'link';
|
||||
if (tag === 'button') return 'button';
|
||||
if (tag === 'select') return 'combobox';
|
||||
if (tag === 'textarea') return 'textbox';
|
||||
if (tag === 'summary') return 'button';
|
||||
if (tag === 'input') {
|
||||
const type = (element.getAttribute('type') || 'text').toLowerCase();
|
||||
if (type === 'checkbox') return 'checkbox';
|
||||
if (type === 'radio') return 'radio';
|
||||
if (type === 'button' || type === 'submit' || type === 'reset') return 'button';
|
||||
return 'textbox';
|
||||
}
|
||||
return 'generic';
|
||||
}
|
||||
|
||||
function textFor(element) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
const pieces = [
|
||||
element.getAttribute('aria-label'),
|
||||
element.getAttribute('alt'),
|
||||
element.getAttribute('title'),
|
||||
tag === 'input' ? element.getAttribute('placeholder') : null,
|
||||
tag === 'input' || tag === 'textarea' ? element.value : null,
|
||||
element.innerText,
|
||||
element.textContent
|
||||
];
|
||||
const text = pieces.find((piece) => typeof piece === 'string' && piece.trim().length > 0);
|
||||
return (text || '').replace(/\s+/g, ' ').trim().slice(0, 300);
|
||||
}
|
||||
|
||||
function selectorFor(element) {
|
||||
if (element.id) return '#' + cssEscape(element.id);
|
||||
const parts = [];
|
||||
let current = element;
|
||||
while (current && current.nodeType === Node.ELEMENT_NODE && current !== document.body) {
|
||||
const tag = current.tagName.toLowerCase();
|
||||
const parent = current.parentElement;
|
||||
if (!parent) break;
|
||||
const siblings = Array.from(parent.children).filter((sibling) => sibling.tagName === current.tagName);
|
||||
const index = siblings.indexOf(current) + 1;
|
||||
parts.unshift(siblings.length > 1 ? tag + ':nth-of-type(' + index + ')' : tag);
|
||||
current = parent;
|
||||
}
|
||||
return parts.length > 0 ? parts.join(' > ') : element.tagName.toLowerCase();
|
||||
}
|
||||
|
||||
return JSON.stringify(Array.from(document.querySelectorAll(CANDIDATE_SELECTOR))
|
||||
.filter(isVisible)
|
||||
.slice(0, MAX_ELEMENTS)
|
||||
.map((element) => ({
|
||||
role: roleFor(element),
|
||||
tagName: element.tagName.toLowerCase(),
|
||||
text: textFor(element),
|
||||
selector: selectorFor(element),
|
||||
attributes: {
|
||||
...(element.id ? { id: element.id } : {}),
|
||||
...(element.getAttribute('name') ? { name: element.getAttribute('name') } : {}),
|
||||
...(element.getAttribute('type') ? { type: element.getAttribute('type') } : {}),
|
||||
...(element.getAttribute('href') ? { href: element.getAttribute('href') } : {}),
|
||||
...(element.getAttribute('aria-label') ? { 'aria-label': element.getAttribute('aria-label') } : {})
|
||||
}
|
||||
})));
|
||||
})()`;
|
||||
function readNumber(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { dispatchTrustedKey } from "./trusted-input.js";
|
||||
|
||||
describe("trusted browser input", () => {
|
||||
test("Space dispatches a real space key event", async () => {
|
||||
const commands: Array<{ command: string; params?: Record<string, unknown> }> = [];
|
||||
|
||||
await dispatchTrustedKey(async (command, params) => {
|
||||
commands.push({ command, ...(params ? { params } : {}) });
|
||||
return {};
|
||||
}, "Space");
|
||||
|
||||
expect(commands).toEqual([
|
||||
{
|
||||
command: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyDown",
|
||||
key: " ",
|
||||
code: "Space",
|
||||
windowsVirtualKeyCode: 32,
|
||||
nativeVirtualKeyCode: 32,
|
||||
text: " ",
|
||||
unmodifiedText: " ",
|
||||
},
|
||||
},
|
||||
{
|
||||
command: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyUp",
|
||||
key: " ",
|
||||
code: "Space",
|
||||
windowsVirtualKeyCode: 32,
|
||||
nativeVirtualKeyCode: 32,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,217 @@
|
||||
import type { ActionablePoint } from "./actionability.js";
|
||||
import type { CdpCommandSender } from "./cdp-session-queue.js";
|
||||
|
||||
export type MouseButton = "left" | "right" | "middle";
|
||||
export type InputModifier = "Alt" | "Control" | "Meta" | "Shift";
|
||||
|
||||
export interface ClickInputOptions {
|
||||
button?: MouseButton;
|
||||
doubleClick?: boolean;
|
||||
modifiers?: InputModifier[];
|
||||
}
|
||||
|
||||
const MODIFIER_MASKS: Record<InputModifier, number> = {
|
||||
Alt: 1,
|
||||
Control: 2,
|
||||
Meta: 4,
|
||||
Shift: 8,
|
||||
};
|
||||
|
||||
const SPECIAL_KEY_DEFINITIONS: Record<
|
||||
string,
|
||||
{ key: string; code: string; windowsVirtualKeyCode: number; text?: string }
|
||||
> = {
|
||||
Enter: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13, text: "\r" },
|
||||
Space: { key: " ", code: "Space", windowsVirtualKeyCode: 32, text: " " },
|
||||
Tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9, text: "\t" },
|
||||
Escape: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
|
||||
Backspace: { key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 },
|
||||
Delete: { key: "Delete", code: "Delete", windowsVirtualKeyCode: 46 },
|
||||
ArrowUp: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
|
||||
ArrowDown: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
|
||||
ArrowLeft: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
|
||||
ArrowRight: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
|
||||
Home: { key: "Home", code: "Home", windowsVirtualKeyCode: 36 },
|
||||
End: { key: "End", code: "End", windowsVirtualKeyCode: 35 },
|
||||
PageUp: { key: "PageUp", code: "PageUp", windowsVirtualKeyCode: 33 },
|
||||
PageDown: { key: "PageDown", code: "PageDown", windowsVirtualKeyCode: 34 },
|
||||
};
|
||||
|
||||
export async function dispatchTrustedClick(
|
||||
send: CdpCommandSender,
|
||||
point: ActionablePoint,
|
||||
options: ClickInputOptions = {},
|
||||
): Promise<void> {
|
||||
const button = options.button ?? "left";
|
||||
const modifiers = modifierMask(options.modifiers);
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button: "none",
|
||||
modifiers,
|
||||
});
|
||||
if (options.doubleClick) {
|
||||
await dispatchTrustedMouseClick(send, point, button, modifiers, 1);
|
||||
await dispatchTrustedMouseClick(send, point, button, modifiers, 2);
|
||||
return;
|
||||
}
|
||||
await dispatchTrustedMouseClick(send, point, button, modifiers, 1);
|
||||
}
|
||||
|
||||
async function dispatchTrustedMouseClick(
|
||||
send: CdpCommandSender,
|
||||
point: ActionablePoint,
|
||||
button: MouseButton,
|
||||
modifiers: number,
|
||||
clickCount: number,
|
||||
): Promise<void> {
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
buttons: mouseButtonMask(button),
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
buttons: 0,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
}
|
||||
|
||||
export async function dispatchTrustedHover(
|
||||
send: CdpCommandSender,
|
||||
point: ActionablePoint,
|
||||
): Promise<void> {
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button: "none",
|
||||
});
|
||||
}
|
||||
|
||||
export async function dispatchTrustedDrag(
|
||||
send: CdpCommandSender,
|
||||
source: ActionablePoint,
|
||||
target: ActionablePoint,
|
||||
): Promise<void> {
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: source.x,
|
||||
y: source.y,
|
||||
button: "none",
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: source.x,
|
||||
y: source.y,
|
||||
button: "left",
|
||||
buttons: 1,
|
||||
clickCount: 1,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: (source.x + target.x) / 2,
|
||||
y: (source.y + target.y) / 2,
|
||||
button: "left",
|
||||
buttons: 1,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: target.x,
|
||||
y: target.y,
|
||||
button: "left",
|
||||
buttons: 1,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: target.x,
|
||||
y: target.y,
|
||||
button: "left",
|
||||
buttons: 0,
|
||||
clickCount: 1,
|
||||
});
|
||||
}
|
||||
|
||||
export async function dispatchTrustedScroll(
|
||||
send: CdpCommandSender,
|
||||
point: ActionablePoint,
|
||||
deltaX: number,
|
||||
deltaY: number,
|
||||
): Promise<void> {
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseWheel",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
deltaX,
|
||||
deltaY,
|
||||
});
|
||||
}
|
||||
|
||||
export async function dispatchTrustedText(send: CdpCommandSender, text: string): Promise<void> {
|
||||
if (text.length === 0) {
|
||||
return;
|
||||
}
|
||||
await send("Input.insertText", { text });
|
||||
}
|
||||
|
||||
export async function dispatchTrustedKey(send: CdpCommandSender, key: string): Promise<void> {
|
||||
const definition = keyDefinition(key);
|
||||
await send("Input.dispatchKeyEvent", {
|
||||
type: "keyDown",
|
||||
key: definition.key,
|
||||
code: definition.code,
|
||||
windowsVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
nativeVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
...(definition.text ? { text: definition.text, unmodifiedText: definition.text } : {}),
|
||||
});
|
||||
await send("Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
key: definition.key,
|
||||
code: definition.code,
|
||||
windowsVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
nativeVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
});
|
||||
}
|
||||
|
||||
function keyDefinition(key: string): {
|
||||
key: string;
|
||||
code: string;
|
||||
windowsVirtualKeyCode: number;
|
||||
text?: string;
|
||||
} {
|
||||
const special = SPECIAL_KEY_DEFINITIONS[key];
|
||||
if (special) {
|
||||
return special;
|
||||
}
|
||||
const text = key.length === 1 ? key : "";
|
||||
const upper = text.toUpperCase();
|
||||
return {
|
||||
key,
|
||||
code: upper ? `Key${upper}` : key,
|
||||
windowsVirtualKeyCode: upper ? upper.charCodeAt(0) : 0,
|
||||
...(text ? { text, unmodifiedText: text } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function modifierMask(modifiers: InputModifier[] | undefined): number {
|
||||
return (modifiers ?? []).reduce((mask, modifier) => mask | MODIFIER_MASKS[modifier], 0);
|
||||
}
|
||||
|
||||
function mouseButtonMask(button: MouseButton): number {
|
||||
if (button === "right") {
|
||||
return 2;
|
||||
}
|
||||
if (button === "middle") {
|
||||
return 4;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
@@ -70,6 +70,10 @@ export function registerPaseoBrowserWorkspace(input: BrowserWorkspaceRegistratio
|
||||
browserRegistry.registerWorkspace(input);
|
||||
}
|
||||
|
||||
export function unregisterPaseoBrowser(browserId: string): void {
|
||||
browserRegistry.unregisterBrowser(browserId);
|
||||
}
|
||||
|
||||
export function getPaseoBrowserWorkspaceId(browserId: string): string | null {
|
||||
return browserRegistry.getWorkspaceId(browserId);
|
||||
}
|
||||
|
||||
@@ -51,6 +51,16 @@ export class PaseoBrowserWebviewRegistry {
|
||||
this.workspaceIdsByBrowserId.set(input.browserId, input.workspaceId);
|
||||
}
|
||||
|
||||
public unregisterBrowser(browserId: string): void {
|
||||
const webContentsId = this.webContentsIdsByBrowserId.get(browserId) ?? null;
|
||||
if (webContentsId !== null) {
|
||||
this.browserIdsByWebContentsId.delete(webContentsId);
|
||||
this.webContentsIdsByBrowserId.delete(browserId);
|
||||
}
|
||||
this.workspaceIdsByBrowserId.delete(browserId);
|
||||
this.deleteActiveBrowserReferences(browserId);
|
||||
}
|
||||
|
||||
public getWorkspaceId(browserId: string): string | null {
|
||||
return this.workspaceIdsByBrowserId.get(browserId) ?? null;
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ import {
|
||||
listRegisteredPaseoBrowserIds,
|
||||
readBrowserIdFromWebviewAttach,
|
||||
registerBrowserWebviewNavigationGuards,
|
||||
unregisterPaseoBrowser,
|
||||
registerPaseoBrowserWorkspace,
|
||||
registerPaseoBrowserWebContents,
|
||||
setWorkspaceActivePaseoBrowserId,
|
||||
@@ -328,6 +329,12 @@ ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: un
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle("paseo:browser:unregister-workspace-browser", (_event, browserId: unknown) => {
|
||||
if (typeof browserId === "string" && browserId.trim().length > 0) {
|
||||
unregisterPaseoBrowser(browserId.trim());
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput: unknown) => {
|
||||
const input = readActiveBrowserInput(rawInput);
|
||||
if (input) {
|
||||
|
||||
@@ -76,6 +76,8 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||
browser: {
|
||||
registerWorkspaceBrowser: (input: { browserId: string; workspaceId: string }) =>
|
||||
ipcRenderer.invoke("paseo:browser:register-workspace-browser", input),
|
||||
unregisterWorkspaceBrowser: (browserId: string) =>
|
||||
ipcRenderer.invoke("paseo:browser:unregister-workspace-browser", browserId),
|
||||
setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) =>
|
||||
ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input),
|
||||
openDevTools: (browserId: string) =>
|
||||
|
||||
Reference in New Issue
Block a user