Browser tools: full accessibility snapshot, trusted input, dialogs, evaluate, and tab controls (#1881)

* feat(browser): replace flat snapshot with ARIA tree and page-side refs * feat(browser): trusted CDP input with actionability waits * feat(browser): auto-handle javascript dialogs * feat(browser): add browser_evaluate * feat(browser): add scroll, resize, and close_tab tools * fix(browser): review fixes for snapshot/input/dialog seams * fix(browser): address review findings on dialog capture, truncation, and keypress targets * fix(harness): drop resize check that cannot run under parked compositor surfaces * fix(browser): second review round — snapshot fidelity, key input, dialog scoping
2026-07-04 01:38:07 +02:00
parent b28f291493
commit 9c220c2451
29 changed files with 5653 additions and 949 deletions
@@ -1,9 +1,14 @@
 import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
 import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
-import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident";
-import { createWorkspaceBrowser } from "@/stores/browser-store";
+import {
+  ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault,
+  removeResidentBrowserWebview,
+  resizeResidentBrowserWebview,
+} from "@/components/browser-webview-resident";
+import { createWorkspaceBrowser, getBrowserRecord, useBrowserStore } from "@/stores/browser-store";
 import {
  buildWorkspaceTabPersistenceKey,
+  collectAllTabs,
  useWorkspaceLayoutStore,
 } from "@/stores/workspace-layout-store";

@@ -114,6 +119,33 @@ async function handleBrowserAutomationRequest(params: {
    return;
  }

+  if (request.command.command === "resize") {
+    client.sendBrowserAutomationExecuteResponse({
+      type: "browser.automation.execute.response",
+      payload: resizeBrowserTabForRequest({ request, serverId }),
+    });
+    return;
+  }
+
+  if (request.command.command === "close_tab") {
+    try {
+      client.sendBrowserAutomationExecuteResponse({
+        type: "browser.automation.execute.response",
+        payload: await closeBrowserTabForRequest({
+          request,
+          serverId,
+          browserHost,
+        }),
+      });
+    } catch (error) {
+      client.sendBrowserAutomationExecuteResponse({
+        type: "browser.automation.execute.response",
+        payload: normalizeThrownBridgeError(request.requestId, error),
+      });
+    }
+    return;
+  }
+
  if (!executeAutomationCommand) {
    client.sendBrowserAutomationExecuteResponse({
      type: "browser.automation.execute.response",
@@ -140,6 +172,127 @@ async function handleBrowserAutomationRequest(params: {
  }
 }

+function resizeBrowserTabForRequest(params: {
+  request: BrowserAutomationExecuteRequest;
+  serverId?: string;
+}): BrowserAutomationResponsePayload {
+  const { request, serverId } = params;
+  const command = request.command as Extract<
+    BrowserAutomationExecuteRequest["command"],
+    { command: "resize" }
+  >;
+  const browserId = command.args.browserId;
+  if (!getBrowserRecord(browserId)) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_tab_not_found",
+      message: `No browser tab found for ID: ${browserId}`,
+    });
+  }
+
+  const workspaceId = request.workspaceId;
+  if (serverId && workspaceId && !findWorkspaceBrowserTab({ serverId, workspaceId, browserId })) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_tab_not_found",
+      message: `No browser tab found for ID: ${browserId}`,
+    });
+  }
+
+  const dimensions = resizeResidentBrowserWebview({
+    browserId,
+    width: command.args.width,
+    height: command.args.height,
+  });
+  if (!dimensions) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_tab_not_found",
+      message: `No browser tab found for ID: ${browserId}`,
+    });
+  }
+
+  return {
+    requestId: request.requestId,
+    ok: true,
+    result: {
+      command: "resize",
+      browserId,
+      width: dimensions.width,
+      height: dimensions.height,
+    },
+  };
+}
+
+async function closeBrowserTabForRequest(params: {
+  request: BrowserAutomationExecuteRequest;
+  serverId?: string;
+  browserHost: DesktopHostBridge["browser"] | undefined;
+}): Promise<BrowserAutomationResponsePayload> {
+  const { request, serverId, browserHost } = params;
+  const command = request.command as Extract<
+    BrowserAutomationExecuteRequest["command"],
+    { command: "close_tab" }
+  >;
+  const browserId = command.args.browserId;
+  const workspaceId = request.workspaceId;
+  const workspaceTab = serverId
+    ? findWorkspaceBrowserTab({ serverId, workspaceId, browserId })
+    : null;
+  if (!workspaceTab && (!serverId || !workspaceId)) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_unsupported",
+      message: "Cannot close a browser tab without a workspace context.",
+    });
+  }
+  if (!workspaceTab || !getBrowserRecord(browserId)) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_tab_not_found",
+      message: `No browser tab found for ID: ${browserId}`,
+    });
+  }
+
+  useWorkspaceLayoutStore.getState().closeTab(workspaceTab.workspaceKey, workspaceTab.tabId);
+  useBrowserStore.getState().removeBrowser(browserId);
+  removeResidentBrowserWebview(browserId);
+  await browserHost?.unregisterWorkspaceBrowser?.(browserId);
+  await browserHost?.clearPartition?.(browserId);
+
+  return {
+    requestId: request.requestId,
+    ok: true,
+    result: { command: "close_tab", browserId },
+  };
+}
+
+function findWorkspaceBrowserTab(input: {
+  serverId: string;
+  workspaceId: string | undefined;
+  browserId: string;
+}): { workspaceKey: string; tabId: string } | null {
+  if (!input.workspaceId) {
+    return null;
+  }
+  const workspaceKey = buildWorkspaceTabPersistenceKey({
+    serverId: input.serverId,
+    workspaceId: input.workspaceId,
+  });
+  if (!workspaceKey) {
+    return null;
+  }
+  const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey];
+  const tab = layout
+    ? collectAllTabs(layout.root).find((candidate) => {
+        return (
+          candidate.target.kind === "browser" && candidate.target.browserId === input.browserId
+        );
+      })
+    : null;
+  return tab ? { workspaceKey, tabId: tab.tabId } : null;
+}
+
 async function openBrowserTabForRequest(params: {
  request: BrowserAutomationExecuteRequest;
  serverId?: string;