Browser tools: full accessibility snapshot, trusted input, dialogs, evaluate, and tab controls (#1881)

* feat(browser): replace flat snapshot with ARIA tree and page-side refs * feat(browser): trusted CDP input with actionability waits * feat(browser): auto-handle javascript dialogs * feat(browser): add browser_evaluate * feat(browser): add scroll, resize, and close_tab tools * fix(browser): review fixes for snapshot/input/dialog seams * fix(browser): address review findings on dialog capture, truncation, and keypress targets * fix(harness): drop resize check that cannot run under parked compositor surfaces * fix(browser): second review round — snapshot fidelity, key input, dialog scoping
2026-07-04 01:38:07 +02:00
parent b28f291493
commit 9c220c2451
29 changed files with 5653 additions and 949 deletions
@@ -1,8 +1,20 @@
 import type { Rectangle } from "electron";
 import { ipcMain } from "electron";
 import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
-import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
+import type {
+  BrowserAutomationConsoleLogEntry,
+  BrowserAutomationDialogEvent,
+} from "@getpaseo/protocol/browser-automation/rpc-schemas";
 import type { TabContents, BrowserRegistry, TabImage } from "./service.js";
+import { CdpSessionQueue } from "./cdp-session-queue.js";
+import {
+  dialogAcceptValue,
+  handledDialogEvent,
+  MAX_DIALOGS_PER_COMMAND,
+  promptShimDrainScript,
+  promptShimInstallScript,
+  promptShimRestoreScript,
+} from "./dialog-handling.js";
 import { executeAutomationCommand } from "./service.js";
 import {
  listRegisteredPaseoBrowserIds,
@@ -14,6 +26,8 @@ import {

 const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
 const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
+const cdpQueuesByContentsId = new Map<number, CdpSessionQueue>();
+const dialogMonitorsByContentsId = new Map<number, DialogMonitor>();
 const observedContentsIds = new Set<number>();

 interface IpcHandlerRegistry {
@@ -24,6 +38,10 @@ interface WebContentsDebugger {
  isAttached(): boolean;
  attach(protocolVersion?: string): void;
  sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown>;
+  on?(
+    event: "message",
+    listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
+  ): void;
 }

 interface ConsoleMessageEmitter {
@@ -60,6 +78,8 @@ interface BrowserAutomationWebContents extends ConsoleMessageEmitter {

 export function adaptWebContents(contents: BrowserAutomationWebContents): TabContents {
  observeConsoleMessages(contents);
+  const cdpQueue = getCdpQueue(contents.id);
+  const dialogMonitor = getDialogMonitor(contents, cdpQueue);
  return {
    id: contents.id,
    getURL: () => contents.getURL(),
@@ -76,15 +96,27 @@ export function adaptWebContents(contents: BrowserAutomationWebContents): TabCon
    capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions),
    invalidate: () => contents.invalidate(),
    getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
-    sendDebugCommand: async (command: string, params?: Record<string, unknown>) => {
-      if (!contents.debugger.isAttached()) {
-        contents.debugger.attach("1.3");
-      }
-      return contents.debugger.sendCommand(command, params ?? {});
-    },
+    captureDialogs: (task) => dialogMonitor.capture(task),
+    sendDebugCommand: (command: string, params?: Record<string, unknown>) =>
+      cdpQueue.run(async () => {
+        if (!contents.debugger.isAttached()) {
+          contents.debugger.attach("1.3");
+        }
+        return contents.debugger.sendCommand(command, params ?? {});
+      }),
  };
 }

+function getCdpQueue(contentsId: number): CdpSessionQueue {
+  const existing = cdpQueuesByContentsId.get(contentsId);
+  if (existing) {
+    return existing;
+  }
+  const queue = new CdpSessionQueue();
+  cdpQueuesByContentsId.set(contentsId, queue);
+  return queue;
+}
+
 function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
  if (observedContentsIds.has(contents.id)) {
    return;
@@ -99,6 +131,192 @@ function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
  contents.once("destroyed", () => {
    observedContentsIds.delete(contents.id);
    consoleMessagesByContentsId.delete(contents.id);
+    cdpQueuesByContentsId.delete(contents.id);
+    dialogMonitorsByContentsId.delete(contents.id);
+  });
+}
+
+function getDialogMonitor(
+  contents: BrowserAutomationWebContents,
+  cdpQueue: CdpSessionQueue,
+): DialogMonitor {
+  const existing = dialogMonitorsByContentsId.get(contents.id);
+  if (existing) {
+    return existing;
+  }
+  const monitor = new DialogMonitor(contents, cdpQueue);
+  dialogMonitorsByContentsId.set(contents.id, monitor);
+  return monitor;
+}
+
+class DialogMonitor {
+  private enabled = false;
+  private listenerRegistered = false;
+  private readonly activeCollectors: DialogCollector[] = [];
+
+  public constructor(
+    private readonly contents: BrowserAutomationWebContents,
+    private readonly cdpQueue: CdpSessionQueue,
+  ) {}
+
+  public async capture<T>(
+    task: () => Promise<T>,
+  ): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }> {
+    const collector: DialogCollector = { dialogs: [] };
+    try {
+      await this.enable();
+      await this.installPromptShim();
+    } catch (error) {
+      console.warn("[browser-automation] Dialog capture unavailable; running command without it", {
+        contentsId: this.contents.id,
+        error,
+      });
+      return { result: await task(), dialogs: [] };
+    }
+    this.activeCollectors.push(collector);
+    try {
+      const result = await task();
+      this.recordPromptShimDialogs(await this.drainPromptShim());
+      return { result, dialogs: collector.dialogs };
+    } finally {
+      const index = this.activeCollectors.indexOf(collector);
+      if (index >= 0) {
+        this.activeCollectors.splice(index, 1);
+      }
+      if (this.activeCollectors.length === 0) {
+        await this.restorePromptShim();
+      }
+    }
+  }
+
+  private async enable(): Promise<void> {
+    if (this.enabled) {
+      return;
+    }
+    if (!this.contents.debugger.on) {
+      return;
+    }
+    if (!this.listenerRegistered) {
+      this.listenerRegistered = true;
+      this.contents.debugger.on("message", (_event, method, params) => {
+        if (method !== "Page.javascriptDialogOpening") {
+          return;
+        }
+        if (this.activeCollectors.length === 0) {
+          return;
+        }
+        void this.handleOpening(params ?? {});
+      });
+    }
+    await this.sendDebugCommand("Page.enable");
+    this.enabled = true;
+  }
+
+  private async handleOpening(params: Record<string, unknown>): Promise<void> {
+    const event = handledDialogEvent(params);
+    for (const collector of this.activeCollectors) {
+      this.recordDialogs(collector, [event]);
+    }
+    await this.sendDialogResponseCommand("Page.handleJavaScriptDialog", {
+      accept: dialogAcceptValue(event.type),
+    });
+  }
+
+  private async installPromptShim(): Promise<void> {
+    await this.sendDebugCommand("Runtime.evaluate", {
+      expression: promptShimInstallScript(),
+      returnByValue: true,
+    });
+  }
+
+  private async drainPromptShim(): Promise<BrowserAutomationDialogEvent[]> {
+    try {
+      const result = (await this.sendDebugCommand("Runtime.evaluate", {
+        expression: promptShimDrainScript(),
+        returnByValue: true,
+      })) as { result?: { value?: unknown } };
+      return parsePromptShimDialogs(result.result?.value);
+    } catch {
+      return [];
+    }
+  }
+
+  private async restorePromptShim(): Promise<void> {
+    try {
+      await this.sendDebugCommand("Runtime.evaluate", {
+        expression: promptShimRestoreScript(),
+        returnByValue: true,
+      });
+    } catch {
+      // Navigation can destroy the execution context before cleanup runs; the next page has no shim.
+    }
+  }
+
+  private recordDialogs(collector: DialogCollector, dialogs: BrowserAutomationDialogEvent[]): void {
+    for (const dialog of dialogs) {
+      if (collector.dialogs.length >= MAX_DIALOGS_PER_COMMAND) {
+        return;
+      }
+      collector.dialogs.push(dialog);
+    }
+  }
+
+  private recordPromptShimDialogs(dialogs: BrowserAutomationDialogEvent[]): void {
+    for (const collector of this.activeCollectors) {
+      this.recordDialogs(collector, dialogs);
+    }
+  }
+
+  private async sendDebugCommand(
+    command: string,
+    params?: Record<string, unknown>,
+  ): Promise<unknown> {
+    return this.cdpQueue.run(async () => {
+      if (!this.contents.debugger.isAttached()) {
+        this.contents.debugger.attach("1.3");
+      }
+      return this.contents.debugger.sendCommand(command, params ?? {});
+    });
+  }
+
+  private async sendDialogResponseCommand(
+    command: string,
+    params?: Record<string, unknown>,
+  ): Promise<unknown> {
+    // Dialogs can block the CDP command that opened them, so the unblocker must not wait behind
+    // the per-tab command queue.
+    if (!this.contents.debugger.isAttached()) {
+      this.contents.debugger.attach("1.3");
+    }
+    return this.contents.debugger.sendCommand(command, params ?? {});
+  }
+}
+
+interface DialogCollector {
+  dialogs: BrowserAutomationDialogEvent[];
+}
+
+function parsePromptShimDialogs(value: unknown): BrowserAutomationDialogEvent[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  return value.flatMap((entry): BrowserAutomationDialogEvent[] => {
+    if (!entry || typeof entry !== "object") {
+      return [];
+    }
+    const record = entry as Record<string, unknown>;
+    if (record.type !== "prompt" || record.action !== "dismissed") {
+      return [];
+    }
+    return [
+      {
+        type: "prompt",
+        message: typeof record.message === "string" ? record.message : "",
+        ...(typeof record.defaultValue === "string" ? { defaultValue: record.defaultValue } : {}),
+        action: "dismissed",
+        timestamp: typeof record.timestamp === "number" ? record.timestamp : Date.now(),
+      },
+    ];
  });
 }