Browser tools: full accessibility snapshot, trusted input, dialogs, evaluate, and tab controls (#1881)

* feat(browser): replace flat snapshot with ARIA tree and page-side refs * feat(browser): trusted CDP input with actionability waits * feat(browser): auto-handle javascript dialogs * feat(browser): add browser_evaluate * feat(browser): add scroll, resize, and close_tab tools * fix(browser): review fixes for snapshot/input/dialog seams * fix(browser): address review findings on dialog capture, truncation, and keypress targets * fix(harness): drop resize check that cannot run under parked compositor surfaces * fix(browser): second review round — snapshot fidelity, key input, dialog scoping
2026-07-04 01:38:07 +02:00
parent b28f291493
commit 9c220c2451
29 changed files with 5653 additions and 949 deletions
@@ -75,6 +75,8 @@ class FakeBrowserBridge {
  public readonly executedRequests: BrowserAutomationExecuteRequest[] = [];
  public readonly registeredWorkspaceBrowsers: Array<{ browserId: string; workspaceId: string }> =
    [];
+  public readonly unregisteredWorkspaceBrowsers: string[] = [];
+  public readonly clearedPartitions: string[] = [];
  public readonly activeWorkspaceBrowsers: Array<{
    browserId: string | null;
    workspaceId: string;
@@ -99,6 +101,14 @@ class FakeBrowserBridge {
    this.registeredWorkspaceBrowsers.push(input);
  };

+  public unregisterWorkspaceBrowser = async (browserId: string): Promise<void> => {
+    this.unregisteredWorkspaceBrowsers.push(browserId);
+  };
+
+  public clearPartition = async (browserId: string): Promise<void> => {
+    this.clearedPartitions.push(browserId);
+  };
+
  public setWorkspaceActiveBrowser = async (input: {
    browserId: string | null;
    workspaceId: string;
@@ -175,6 +185,35 @@ function browserNewTabRequest(): BrowserAutomationExecuteRequest {
  };
 }

+function browserResizeRequest(
+  browserId: string,
+  input: { workspaceId?: string } = {},
+): BrowserAutomationExecuteRequest {
+  return {
+    type: "browser.automation.execute.request",
+    requestId: "req-resize",
+    agentId: "agent-1",
+    workspaceId: input.workspaceId ?? "wks_workspace_a",
+    command: {
+      command: "resize",
+      args: { browserId, width: 1024, height: 768 },
+    },
+  };
+}
+
+function browserCloseTabRequest(browserId: string): BrowserAutomationExecuteRequest {
+  return {
+    type: "browser.automation.execute.request",
+    requestId: "req-close-tab",
+    agentId: "agent-1",
+    workspaceId: "wks_workspace_a",
+    command: {
+      command: "close_tab",
+      args: { browserId },
+    },
+  };
+}
+
 function emptyListTabsPayload(requestId = "req-new:list_tabs"): BrowserAutomationResponsePayload {
  return {
    requestId,
@@ -355,6 +394,106 @@ describe("mountBrowserAutomationHandler", () => {
    ]);
  });

+  test("browser_resize updates resident webview dimensions", async () => {
+    const browser = new BrowserAutomationHandlerHarness();
+    browser.mount({ serverId: "server-1" });
+
+    browser.receive(browserNewTabRequest());
+    await flushAsyncWork();
+    const result = newTabResultFrom(browser.client.payloadAt(0));
+
+    browser.receive(browserResizeRequest(result.browserId));
+    await flushAsyncWork();
+
+    expect(browser.client.payloadAt(1)).toEqual({
+      requestId: "req-resize",
+      ok: true,
+      result: {
+        command: "resize",
+        browserId: result.browserId,
+        width: 1024,
+        height: 768,
+      },
+    });
+    expect(browser.browser.executedRequests).toHaveLength(1);
+  });
+
+  test("browser_resize returns not found for a tab outside the request workspace", async () => {
+    const browser = new BrowserAutomationHandlerHarness();
+    browser.mount({ serverId: "server-1" });
+
+    browser.receive(browserNewTabRequest());
+    await flushAsyncWork();
+    const result = newTabResultFrom(browser.client.payloadAt(0));
+
+    browser.receive(browserResizeRequest(result.browserId, { workspaceId: "wks_workspace_b" }));
+    await flushAsyncWork();
+
+    expect(browser.client.payloadAt(1)).toEqual({
+      requestId: "req-resize",
+      ok: false,
+      error: {
+        code: "browser_tab_not_found",
+        message: `No browser tab found for ID: ${result.browserId}`,
+        retryable: false,
+      },
+    });
+  });
+
+  test("browser_close_tab removes the workspace tab, browser record, resident webview, registry entry, and partition", async () => {
+    const browser = new BrowserAutomationHandlerHarness();
+    const workspaceKey = buildWorkspaceTabPersistenceKey({
+      serverId: "server-1",
+      workspaceId: "wks_workspace_a",
+    });
+    if (!workspaceKey) {
+      throw new Error("Expected workspace key");
+    }
+    browser.mount({ serverId: "server-1" });
+
+    browser.receive(browserNewTabRequest());
+    await flushAsyncWork();
+    const result = newTabResultFrom(browser.client.payloadAt(0));
+
+    browser.receive(browserCloseTabRequest(result.browserId));
+    await flushAsyncWork();
+
+    expect(browser.client.payloadAt(1)).toEqual({
+      requestId: "req-close-tab",
+      ok: true,
+      result: { command: "close_tab", browserId: result.browserId },
+    });
+    expect(workspaceBrowserTabs(workspaceKey, result.browserId)).toEqual([]);
+    expect(useBrowserStore.getState().browsersById[result.browserId]).toBeUndefined();
+    expect(browser.browser.unregisteredWorkspaceBrowsers).toEqual([result.browserId]);
+    expect(browser.browser.clearedPartitions).toEqual([result.browserId]);
+    expect(currentBrowserTabs()).toEqual([]);
+  });
+
+  test("browser_close_tab returns not found after the tab is gone", async () => {
+    const browser = new BrowserAutomationHandlerHarness();
+    browser.mount({ serverId: "server-1" });
+
+    browser.receive(browserNewTabRequest());
+    await flushAsyncWork();
+    const result = newTabResultFrom(browser.client.payloadAt(0));
+
+    browser.receive(browserCloseTabRequest(result.browserId));
+    await flushAsyncWork();
+    browser.receive(browserCloseTabRequest(result.browserId));
+    await flushAsyncWork();
+
+    expect(browser.client.payloadAt(2)).toEqual({
+      requestId: "req-close-tab",
+      ok: false,
+      error: {
+        code: "browser_tab_not_found",
+        message: `No browser tab found for ID: ${result.browserId}`,
+        retryable: false,
+      },
+    });
+  });
+
  test("non-new-tab requests send the desktop bridge response", async () => {
    const browser = new BrowserAutomationHandlerHarness();
    browser.browser.response = {
@@ -1,9 +1,14 @@
 import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
 import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
-import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident";
-import { createWorkspaceBrowser } from "@/stores/browser-store";
+import {
+  ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault,
+  removeResidentBrowserWebview,
+  resizeResidentBrowserWebview,
+} from "@/components/browser-webview-resident";
+import { createWorkspaceBrowser, getBrowserRecord, useBrowserStore } from "@/stores/browser-store";
 import {
  buildWorkspaceTabPersistenceKey,
+  collectAllTabs,
  useWorkspaceLayoutStore,
 } from "@/stores/workspace-layout-store";

@@ -114,6 +119,33 @@ async function handleBrowserAutomationRequest(params: {
    return;
  }

+  if (request.command.command === "resize") {
+    client.sendBrowserAutomationExecuteResponse({
+      type: "browser.automation.execute.response",
+      payload: resizeBrowserTabForRequest({ request, serverId }),
+    });
+    return;
+  }
+
+  if (request.command.command === "close_tab") {
+    try {
+      client.sendBrowserAutomationExecuteResponse({
+        type: "browser.automation.execute.response",
+        payload: await closeBrowserTabForRequest({
+          request,
+          serverId,
+          browserHost,
+        }),
+      });
+    } catch (error) {
+      client.sendBrowserAutomationExecuteResponse({
+        type: "browser.automation.execute.response",
+        payload: normalizeThrownBridgeError(request.requestId, error),
+      });
+    }
+    return;
+  }
+
  if (!executeAutomationCommand) {
    client.sendBrowserAutomationExecuteResponse({
      type: "browser.automation.execute.response",
@@ -140,6 +172,127 @@ async function handleBrowserAutomationRequest(params: {
  }
 }

+function resizeBrowserTabForRequest(params: {
+  request: BrowserAutomationExecuteRequest;
+  serverId?: string;
+}): BrowserAutomationResponsePayload {
+  const { request, serverId } = params;
+  const command = request.command as Extract<
+    BrowserAutomationExecuteRequest["command"],
+    { command: "resize" }
+  >;
+  const browserId = command.args.browserId;
+  if (!getBrowserRecord(browserId)) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_tab_not_found",
+      message: `No browser tab found for ID: ${browserId}`,
+    });
+  }
+
+  const workspaceId = request.workspaceId;
+  if (serverId && workspaceId && !findWorkspaceBrowserTab({ serverId, workspaceId, browserId })) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_tab_not_found",
+      message: `No browser tab found for ID: ${browserId}`,
+    });
+  }
+
+  const dimensions = resizeResidentBrowserWebview({
+    browserId,
+    width: command.args.width,
+    height: command.args.height,
+  });
+  if (!dimensions) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_tab_not_found",
+      message: `No browser tab found for ID: ${browserId}`,
+    });
+  }
+
+  return {
+    requestId: request.requestId,
+    ok: true,
+    result: {
+      command: "resize",
+      browserId,
+      width: dimensions.width,
+      height: dimensions.height,
+    },
+  };
+}
+
+async function closeBrowserTabForRequest(params: {
+  request: BrowserAutomationExecuteRequest;
+  serverId?: string;
+  browserHost: DesktopHostBridge["browser"] | undefined;
+}): Promise<BrowserAutomationResponsePayload> {
+  const { request, serverId, browserHost } = params;
+  const command = request.command as Extract<
+    BrowserAutomationExecuteRequest["command"],
+    { command: "close_tab" }
+  >;
+  const browserId = command.args.browserId;
+  const workspaceId = request.workspaceId;
+  const workspaceTab = serverId
+    ? findWorkspaceBrowserTab({ serverId, workspaceId, browserId })
+    : null;
+  if (!workspaceTab && (!serverId || !workspaceId)) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_unsupported",
+      message: "Cannot close a browser tab without a workspace context.",
+    });
+  }
+  if (!workspaceTab || !getBrowserRecord(browserId)) {
+    return browserAutomationFailure({
+      requestId: request.requestId,
+      code: "browser_tab_not_found",
+      message: `No browser tab found for ID: ${browserId}`,
+    });
+  }
+
+  useWorkspaceLayoutStore.getState().closeTab(workspaceTab.workspaceKey, workspaceTab.tabId);
+  useBrowserStore.getState().removeBrowser(browserId);
+  removeResidentBrowserWebview(browserId);
+  await browserHost?.unregisterWorkspaceBrowser?.(browserId);
+  await browserHost?.clearPartition?.(browserId);
+
+  return {
+    requestId: request.requestId,
+    ok: true,
+    result: { command: "close_tab", browserId },
+  };
+}
+
+function findWorkspaceBrowserTab(input: {
+  serverId: string;
+  workspaceId: string | undefined;
+  browserId: string;
+}): { workspaceKey: string; tabId: string } | null {
+  if (!input.workspaceId) {
+    return null;
+  }
+  const workspaceKey = buildWorkspaceTabPersistenceKey({
+    serverId: input.serverId,
+    workspaceId: input.workspaceId,
+  });
+  if (!workspaceKey) {
+    return null;
+  }
+  const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey];
+  const tab = layout
+    ? collectAllTabs(layout.root).find((candidate) => {
+        return (
+          candidate.target.kind === "browser" && candidate.target.browserId === input.browserId
+        );
+      })
+    : null;
+  return tab ? { workspaceKey, tabId: tab.tabId } : null;
+}
+
 async function openBrowserTabForRequest(params: {
  request: BrowserAutomationExecuteRequest;
  serverId?: string;
@@ -4,6 +4,7 @@ const RESIDENT_VIEWPORT_WIDTH = 1280;
 const RESIDENT_VIEWPORT_HEIGHT = 800;

 const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
+const residentWebviewSizesByBrowserId = new Map<string, { width: number; height: number }>();

 interface BrowserWebviewElement extends HTMLElement {
  src: string;
@@ -66,11 +67,24 @@ function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLEle
  return null;
 }

-function applyResidentWebviewStyle(webview: HTMLElement): void {
+function dimensionsForBrowser(browserId: string | null): { width: number; height: number } {
+  if (!browserId) {
+    return { width: RESIDENT_VIEWPORT_WIDTH, height: RESIDENT_VIEWPORT_HEIGHT };
+  }
+  return (
+    residentWebviewSizesByBrowserId.get(browserId) ?? {
+      width: RESIDENT_VIEWPORT_WIDTH,
+      height: RESIDENT_VIEWPORT_HEIGHT,
+    }
+  );
+}
+
+function applyResidentWebviewStyle(webview: HTMLElement, browserId: string | null): void {
+  const dimensions = dimensionsForBrowser(browserId);
  webview.style.display = "inline-flex";
  webview.style.flex = "0 0 auto";
-  webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
-  webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
+  webview.style.width = `${dimensions.width}px`;
+  webview.style.height = `${dimensions.height}px`;
  webview.style.border = "0";
  webview.style.background = "transparent";
  webview.style.position = "absolute";
@@ -163,10 +177,33 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl
  }

  residentWebviewsByBrowserId.set(normalizedBrowserId, webview);
-  applyResidentWebviewStyle(webview);
+  applyResidentWebviewStyle(webview, normalizedBrowserId);
  getResidentBrowserHost(ownerDocument).appendChild(webview);
 }

+export function resizeResidentBrowserWebview(input: {
+  browserId: string;
+  width: number;
+  height: number;
+}): { width: number; height: number } | null {
+  const normalizedBrowserId = trimNonEmpty(input.browserId);
+  if (!normalizedBrowserId) {
+    return null;
+  }
+  const width = Math.max(1, Math.round(input.width));
+  const height = Math.max(1, Math.round(input.height));
+  residentWebviewSizesByBrowserId.set(normalizedBrowserId, { width, height });
+
+  const ownerDocument = readDocument();
+  const webview = ownerDocument ? findBrowserWebview(normalizedBrowserId, ownerDocument) : null;
+  if (webview) {
+    webview.style.width = `${width}px`;
+    webview.style.height = `${height}px`;
+  }
+
+  return { width, height };
+}
+
 export function removeResidentBrowserWebview(browserId: string): void {
  const normalizedBrowserId = trimNonEmpty(browserId);
  if (!normalizedBrowserId) {
@@ -175,6 +212,7 @@ export function removeResidentBrowserWebview(browserId: string): void {

  const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
  residentWebviewsByBrowserId.delete(normalizedBrowserId);
+  residentWebviewSizesByBrowserId.delete(normalizedBrowserId);
  resident?.remove();
 }

@@ -183,5 +221,6 @@ export function clearResidentBrowserWebviewsForTests(): void {
    webview.remove();
  }
  residentWebviewsByBrowserId.clear();
+  residentWebviewSizesByBrowserId.clear();
  readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
 }
@@ -128,6 +128,7 @@ export interface DesktopBrowserNewTabRequestEvent {

 export interface DesktopBrowserBridge {
  registerWorkspaceBrowser?: (input: { browserId: string; workspaceId: string }) => Promise<void>;
+  unregisterWorkspaceBrowser?: (browserId: string) => Promise<void>;
  setWorkspaceActiveBrowser?: (input: {
    workspaceId: string;
    browserId: string | null;