From 2ba3ad53ab3ada8d7d3efb7f0627f075757e5c67 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Thu, 2 Jul 2026 23:56:20 +0200 Subject: [PATCH] feat(browser-tools): cut low-value tools --- .../features/browser-automation/ipc.test.ts | 18 - .../src/features/browser-automation/ipc.ts | 57 +- .../browser-automation/service.test.ts | 263 ++-------- .../features/browser-automation/service.ts | 495 +----------------- .../snapshot-engine.test.ts | 4 +- .../browser-automation/snapshot-engine.ts | 76 --- .../browser-automation/rpc-schemas.test.ts | 222 +------- .../src/browser-automation/rpc-schemas.ts | 200 +------ .../src/messages.browser-automation.test.ts | 2 +- .../src/server/agent/mcp-server.test.ts | 2 +- .../src/server/browser-tools/broker.test.ts | 8 +- .../src/server/browser-tools/tools.test.ts | 227 ++------ .../server/src/server/browser-tools/tools.ts | 341 +----------- 13 files changed, 119 insertions(+), 1796 deletions(-) delete mode 100644 packages/desktop/src/features/browser-automation/ipc.test.ts diff --git a/packages/desktop/src/features/browser-automation/ipc.test.ts b/packages/desktop/src/features/browser-automation/ipc.test.ts deleted file mode 100644 index 81245483a..000000000 --- a/packages/desktop/src/features/browser-automation/ipc.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { sanitizeDownloadFileName } from "./ipc.js"; - -describe("browser automation IPC", () => { - it("strips directories from agent-supplied download filenames", () => { - expect( - sanitizeDownloadFileName({ - url: "https://example.com/fallback.txt", - fileName: "../../.ssh/authorized_keys", - }), - ).toBe("authorized_keys"); - }); - - it("falls back to a safe filename when the URL has no basename", () => { - expect(sanitizeDownloadFileName({ url: "https://example.com/" })).toBe("download"); - }); -}); diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts index 94660a772..06b235c8c 100644 --- a/packages/desktop/src/features/browser-automation/ipc.ts +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -1,13 +1,7 @@ -import { mkdirSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { basename, join } from "node:path"; import type { WebContents } from "electron"; import { ipcMain } from "electron"; import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas"; -import type { - BrowserAutomationConsoleLogEntry, - BrowserAutomationCookieEntry, -} from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import type { TabContents, BrowserRegistry } from "./service.js"; import { executeAutomationCommand } from "./service.js"; import { @@ -46,61 +40,12 @@ function adaptWebContents(contents: WebContents): TabContents { isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(), setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed), getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [], - getCookies: async (url: string) => - (await contents.session.cookies.get({ url })).map(normalizeCookie), sendDebugCommand: async (command: string, params?: Record) => { if (!contents.debugger.isAttached()) { contents.debugger.attach("1.3"); } return contents.debugger.sendCommand(command, params ?? {}); }, - printToPDF: async (options?: Record) => contents.printToPDF(options ?? {}), - downloadURL: (input) => downloadWithContents(contents, input), - }; -} - -function downloadWithContents( - contents: WebContents, - input: { url: string; fileName?: string }, -): Promise<{ filePath: string; totalBytes?: number; state: string }> { - const downloadDir = join(tmpdir(), "paseo-browser-downloads"); - mkdirSync(downloadDir, { recursive: true }); - const filePath = join(downloadDir, sanitizeDownloadFileName(input)); - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - contents.session.off("will-download", onDownload); - reject(new Error(`Timed out waiting for browser download: ${input.url}`)); - }, 30_000); - function onDownload(_event: Electron.Event, item: Electron.DownloadItem): void { - if (item.getURL() !== input.url) { - return; - } - clearTimeout(timeout); - contents.session.off("will-download", onDownload); - item.setSavePath(filePath); - item.once("done", (_doneEvent, state) => { - resolve({ filePath, totalBytes: item.getTotalBytes(), state }); - }); - } - contents.session.on("will-download", onDownload); - contents.downloadURL(input.url); - }); -} - -export function sanitizeDownloadFileName(input: { url: string; fileName?: string }): string { - const requestedName = input.fileName ?? basename(new URL(input.url).pathname); - return basename(requestedName) || "download"; -} - -function normalizeCookie(cookie: Electron.Cookie): BrowserAutomationCookieEntry { - return { - name: cookie.name, - value: cookie.value, - ...(cookie.domain ? { domain: cookie.domain } : {}), - ...(cookie.path ? { path: cookie.path } : {}), - secure: cookie.secure, - httpOnly: cookie.httpOnly, - ...(typeof cookie.expirationDate === "number" ? { expirationDate: cookie.expirationDate } : {}), }; } diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index f53e2deba..f949c3982 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -4,7 +4,6 @@ import { describe, expect, test, vi } from "vitest"; import type { BrowserAutomationCommand, BrowserAutomationConsoleLogEntry, - BrowserAutomationCookieEntry, BrowserAutomationExecuteRequest, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { BrowserSnapshotEngine } from "./snapshot-engine.js"; @@ -37,25 +36,14 @@ class FakeTab implements TabContents { public readonly actions: string[] = []; public readonly capturedViewports: Array<{ stayHidden?: boolean }> = []; public readonly debugCommands: Array<{ command: string; params?: Record }> = []; - public readonly pdfOptions: Record[] = []; - public readonly downloads: Array<{ url: string; fileName?: string }> = []; public destroyed = false; public bodyText = ""; public snapshotElements: unknown[] = []; public actionScriptResult: unknown = true; public networkEntries: unknown[] = []; - public storageState: unknown = { localStorage: [], sessionStorage: [] }; - public viewport = { width: 1024, height: 768, deviceScaleFactor: 2 }; public consoleMessages: BrowserAutomationConsoleLogEntry[] = []; - public cookies: BrowserAutomationCookieEntry[] = []; public captureNeverPaints = false; - public pdfBytes = new Uint8Array([37, 80, 68, 70]); - public downloadResult = { - filePath: "/workspace/downloads/file.txt", - totalBytes: 42, - state: "completed", - }; public layoutMetrics = { cssLayoutViewport: { clientWidth: 390, clientHeight: 844 }, cssContentSize: { width: 390, height: 1200 }, @@ -106,12 +94,6 @@ class FakeTab implements TabContents { if (code.includes("performance.getEntriesByType")) { return JSON.stringify(this.networkEntries); } - if (code.includes("localStorage") && code.includes("sessionStorage")) { - return JSON.stringify(this.storageState); - } - if (code.includes("window.innerWidth")) { - return JSON.stringify(this.viewport); - } return this.actionScriptResult; } @@ -157,10 +139,6 @@ class FakeTab implements TabContents { return this.consoleMessages; } - public async getCookies(_url: string): Promise { - return this.cookies; - } - public async sendDebugCommand( command: string, params?: Record, @@ -180,20 +158,6 @@ class FakeTab implements TabContents { } return {}; } - - public async printToPDF(options?: Record): Promise { - this.pdfOptions.push(options ?? {}); - return this.pdfBytes; - } - - public async downloadURL(input: { url: string; fileName?: string }): Promise<{ - filePath: string; - totalBytes?: number; - state: string; - }> { - this.downloads.push(input); - return this.downloadResult; - } } class FakeRegistry implements BrowserRegistry { @@ -470,50 +434,20 @@ describe("executeAutomationCommand", () => { }); }); - test("page info reads the explicit browser id from command args", () => { - const browser = new BrowserAutomationHarness(); - - const result = executeAutomationCommand( - automationRequest( - { command: "page_info", args: { browserId: BROWSER_A } }, - { requestId: "req-page" }, - ), - browser.registry, - ); - - expect(result).toEqual({ - requestId: "req-page", - ok: true, - result: { - command: "page_info", - tab: { - browserId: BROWSER_A, - workspaceId: WORKSPACE_A, - url: "https://a.test/form", - title: "Fixture", - isActive: true, - isLoading: false, - canGoBack: true, - canGoForward: false, - }, - }, - }); - }); - - test("page info returns tab not found for an id in another workspace", () => { + test("tab commands return tab not found for an id in another workspace", async () => { const registry = new FakeRegistry(); registry.register(BROWSER_A, WORKSPACE_B, new FakeTab(1, "https://a.test", "A")); - const result = executeAutomationCommand( + const result = await executeAutomationCommand( automationRequest( - { command: "page_info", args: { browserId: BROWSER_A } }, - { requestId: "req-page" }, + { command: "snapshot", args: { browserId: BROWSER_A } }, + { requestId: "req-snapshot" }, ), registry, ); expect(result).toEqual({ - requestId: "req-page", + requestId: "req-snapshot", ok: false, error: { code: "browser_tab_not_found", @@ -523,22 +457,22 @@ describe("executeAutomationCommand", () => { }); }); - test("page info returns tab closed for a destroyed explicit tab", () => { + test("tab commands return tab closed for a destroyed explicit tab", async () => { const tab = new FakeTab(1, "https://a.test", "A"); tab.destroyed = true; const registry = new FakeRegistry(); registry.register(BROWSER_A, WORKSPACE_A, tab); - const result = executeAutomationCommand( + const result = await executeAutomationCommand( automationRequest( - { command: "page_info", args: { browserId: BROWSER_A } }, - { requestId: "req-page" }, + { command: "snapshot", args: { browserId: BROWSER_A } }, + { requestId: "req-snapshot" }, ), registry, ); expect(result).toEqual({ - requestId: "req-page", + requestId: "req-snapshot", ok: false, error: { code: "browser_tab_closed", @@ -595,22 +529,6 @@ describe("executeAutomationCommand", () => { expect(containsScript(browser.tab, "#submit", ".click()")).toBe(true); }); - test("set background writes the requested color into the explicit page", async () => { - const browser = new BrowserAutomationHarness(); - - const result = await browser.execute({ - command: "set_background", - args: { browserId: BROWSER_A, color: "red" }, - }); - - expect(result).toEqual({ - requestId: "req-set_background", - ok: true, - result: { command: "set_background", browserId: BROWSER_A, color: "red" }, - }); - expect(containsScript(browser.tab, "document.body.style.background", "red")).toBe(true); - }); - test.each([ { name: "fill updates a ref from the latest snapshot", @@ -618,24 +536,6 @@ describe("executeAutomationCommand", () => { result: { command: "fill", browserId: BROWSER_A, ref: "@e1" }, scriptParts: ["#name", "Ada"], }, - { - name: "focus focuses a ref from the latest snapshot", - command: { command: "focus", args: { browserId: BROWSER_A, ref: "@e1" } }, - result: { command: "focus", browserId: BROWSER_A, ref: "@e1" }, - scriptParts: ["#name", "focus"], - }, - { - name: "clear clears a ref from the latest snapshot", - command: { command: "clear", args: { browserId: BROWSER_A, ref: "@e1" } }, - result: { command: "clear", browserId: BROWSER_A, ref: "@e1" }, - scriptParts: ["#name", "deleteContent"], - }, - { - name: "check sets the requested checked state on a ref", - command: { command: "check", args: { browserId: BROWSER_A, ref: "@e2", checked: false } }, - result: { command: "check", browserId: BROWSER_A, ref: "@e2", checked: false }, - scriptParts: ["#agree", "nextChecked = false"], - }, { name: "select sets the requested value on a ref", command: { command: "select", args: { browserId: BROWSER_A, ref: "@e3", value: "us" } }, @@ -672,6 +572,24 @@ describe("executeAutomationCommand", () => { expect(containsScript(browser.tab, ...scriptParts)).toBe(true); }); + test("fill with an empty string clears a ref through the regular fill path", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotElements = formElements(); + + requireSnapshotRefs(await browser.snapshot()); + const action = await browser.execute({ + command: "fill", + args: { browserId: BROWSER_A, ref: "@e1", value: "" }, + }); + + expect(action).toEqual({ + requestId: "req-fill", + ok: true, + result: { command: "fill", browserId: BROWSER_A, ref: "@e1" }, + }); + expect(containsScript(browser.tab, "#name", 'const nextValue = "";')).toBe(true); + }); + test("refs become stale after navigation changes the tab URL", async () => { const browser = new BrowserAutomationHarness(); browser.tab.snapshotElements = formElements(); @@ -912,71 +830,6 @@ describe("executeAutomationCommand", () => { }); }); - test("storage reads cookies and web storage from the explicit tab", async () => { - const browser = new BrowserAutomationHarness(); - browser.tab.cookies = [ - { name: "theme", value: "dark", domain: "a.test", path: "/", secure: true }, - ]; - browser.tab.storageState = { - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }; - - const result = await browser.execute({ - command: "storage", - args: { browserId: BROWSER_A }, - }); - - expect(result).toEqual({ - requestId: "req-storage", - ok: true, - result: { - command: "storage", - browserId: BROWSER_A, - url: "https://a.test/form", - cookies: [{ name: "theme", value: "dark", domain: "a.test", path: "/", secure: true }], - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }, - }); - }); - - test("environment applies viewport and geolocation before reporting the current viewport", async () => { - const browser = new BrowserAutomationHarness(); - browser.tab.viewport = { width: 390, height: 844, deviceScaleFactor: 3 }; - - const result = await browser.execute({ - command: "environment", - args: { - browserId: BROWSER_A, - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }); - - expect(result).toEqual({ - requestId: "req-environment", - ok: true, - result: { - command: "environment", - browserId: BROWSER_A, - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }); - expect(browser.tab.debugCommands).toEqual([ - { - command: "Emulation.setDeviceMetricsOverride", - params: { width: 390, height: 844, deviceScaleFactor: 3, mobile: false }, - }, - { - command: "Emulation.setGeolocationOverride", - params: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - ]); - expect(containsScript(browser.tab, "navigator", "geolocation")).toBe(true); - }); - test("screenshot serializes the painted viewport and restores throttling", async () => { const browser = new BrowserAutomationHarness(); @@ -1039,19 +892,19 @@ describe("executeAutomationCommand", () => { } }); - test("full page screenshot captures the page content area through CDP", async () => { + test("screenshot with fullPage captures the page content area through CDP", async () => { const browser = new BrowserAutomationHarness(); const result = await browser.execute({ - command: "full_page_screenshot", - args: { browserId: BROWSER_A }, + command: "screenshot", + args: { browserId: BROWSER_A, fullPage: true }, }); expect(result).toEqual({ - requestId: "req-full_page_screenshot", + requestId: "req-screenshot", ok: true, result: { - command: "full_page_screenshot", + command: "screenshot", browserId: BROWSER_A, mimeType: "image/png", dataBase64: "fullPagePng", @@ -1072,56 +925,6 @@ describe("executeAutomationCommand", () => { ]); }); - test("pdf exports the explicit tab with requested print options", async () => { - const browser = new BrowserAutomationHarness(); - - const result = await browser.execute({ - command: "pdf", - args: { browserId: BROWSER_A, landscape: true, printBackground: false }, - }); - - expect(result).toEqual({ - requestId: "req-pdf", - ok: true, - result: { - command: "pdf", - browserId: BROWSER_A, - mimeType: "application/pdf", - dataBase64: "JVBERg==", - }, - }); - expect(browser.tab.pdfOptions).toEqual([{ printBackground: false, landscape: true }]); - }); - - test("download saves the requested HTTP URL through the explicit tab", async () => { - const browser = new BrowserAutomationHarness(); - - const result = await browser.execute({ - command: "download", - args: { - browserId: BROWSER_A, - url: "https://a.test/file.txt", - fileName: "file.txt", - }, - }); - - expect(result).toEqual({ - requestId: "req-download", - ok: true, - result: { - command: "download", - browserId: BROWSER_A, - url: "https://a.test/file.txt", - filePath: "/workspace/downloads/file.txt", - totalBytes: 42, - state: "completed", - }, - }); - expect(browser.tab.downloads).toEqual([ - { url: "https://a.test/file.txt", fileName: "file.txt" }, - ]); - }); - test("upload resolves workspace files before setting them on the file input", async () => { const browser = new BrowserAutomationHarness(); browser.tab.snapshotElements = [ diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index 8e64dae5c..9b0c8b60d 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -3,12 +3,10 @@ import { isAbsolute, relative, resolve as resolvePath } from "node:path"; import type { BrowserAutomationCommand, BrowserAutomationConsoleLogEntry, - BrowserAutomationCookieEntry, BrowserAutomationErrorCode, BrowserAutomationExecuteResponse, BrowserAutomationExecuteRequest, BrowserAutomationNetworkLogEntry, - BrowserAutomationStorageEntry, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { BrowserSnapshotEngine } from "./snapshot-engine.js"; @@ -30,14 +28,7 @@ export interface TabContents { isBackgroundThrottlingAllowed(): boolean; setBackgroundThrottling(allowed: boolean): void; getConsoleMessages?(): BrowserAutomationConsoleLogEntry[]; - getCookies?(url: string): Promise; sendDebugCommand?(command: string, params?: Record): Promise; - printToPDF?(options?: Record): Promise; - downloadURL?(input: { url: string; fileName?: string }): Promise<{ - filePath: string; - totalBytes?: number; - state: string; - }>; } export interface TabImage { @@ -195,10 +186,6 @@ const commandHandlers: Record fail(requestId, "browser_unsupported", "browser_new_tab is handled by the app runtime."), - page_info: ({ command, requestId, workspaceId, registry }) => { - const pageInfoCommand = command as Extract; - return executePageInfo(requestId, workspaceId, pageInfoCommand.args.browserId, registry); - }, snapshot: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const snapshotCommand = command as Extract; return executeSnapshot( @@ -315,37 +302,11 @@ const commandHandlers: Record; - return executeScreenshot(requestId, workspaceId, screenshotCommand.args.browserId, registry); - }, - full_page_screenshot: ({ command, requestId, workspaceId, registry }) => { - const screenshotCommand = command as Extract< - BrowserAutomationCommand, - { command: "full_page_screenshot" } - >; - return executeFullPageScreenshot( + return executeScreenshot( requestId, workspaceId, screenshotCommand.args.browserId, - registry, - ); - }, - pdf: ({ command, requestId, workspaceId, registry }) => { - const pdfCommand = command as Extract; - return executePdf( - requestId, - workspaceId, - pdfCommand.args.browserId, - { landscape: pdfCommand.args.landscape, printBackground: pdfCommand.args.printBackground }, - registry, - ); - }, - download: ({ command, requestId, workspaceId, registry }) => { - const downloadCommand = command as Extract; - return executeDownload( - requestId, - workspaceId, - downloadCommand.args.browserId, - { url: downloadCommand.args.url, fileName: downloadCommand.args.fileName }, + screenshotCommand.args.fullPage, registry, ); }, @@ -361,40 +322,6 @@ const commandHandlers: Record { - const focusCommand = command as Extract; - return executeFocus( - requestId, - workspaceId, - focusCommand.args.browserId, - focusCommand.args.ref, - registry, - snapshotEngine, - ); - }, - clear: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { - const clearCommand = command as Extract; - return executeClear( - requestId, - workspaceId, - clearCommand.args.browserId, - clearCommand.args.ref, - registry, - snapshotEngine, - ); - }, - check: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { - const checkCommand = command as Extract; - return executeCheck( - requestId, - workspaceId, - checkCommand.args.browserId, - checkCommand.args.ref, - checkCommand.args.checked, - registry, - snapshotEngine, - ); - }, select: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const selectCommand = command as Extract; return executeSelect( @@ -440,39 +367,6 @@ const commandHandlers: Record { - const storageCommand = command as Extract; - return executeStorage(requestId, workspaceId, storageCommand.args.browserId, registry); - }, - environment: ({ command, requestId, workspaceId, registry }) => { - const environmentCommand = command as Extract< - BrowserAutomationCommand, - { command: "environment" } - >; - return executeEnvironment( - requestId, - workspaceId, - environmentCommand.args.browserId, - { - viewport: environmentCommand.args.viewport, - geolocation: environmentCommand.args.geolocation, - }, - registry, - ); - }, - set_background: ({ command, requestId, workspaceId, registry }) => { - const setBackgroundCommand = command as Extract< - BrowserAutomationCommand, - { command: "set_background" } - >; - return executeSetBackground( - requestId, - workspaceId, - setBackgroundCommand.args.browserId, - setBackgroundCommand.args.color, - registry, - ); - }, }; interface ResolvedTabTarget { @@ -508,37 +402,6 @@ function executeListTabs( return { requestId, ok: true, result: { command: "list_tabs", tabs } }; } -function executePageInfo( - requestId: string, - workspaceId: string | undefined, - browserId: string, - registry: BrowserRegistry, -): AutomationCommandPayload { - const target = resolveTabTarget({ - requestId, - workspaceId, - browserId, - registry, - }); - if ("ok" in target) { - return target; - } - - return { - requestId, - ok: true, - result: { - command: "page_info", - tab: tabInfoFromContents( - target.browserId, - target.contents, - workspaceId ? registry.getWorkspaceActiveBrowserId(workspaceId) : null, - registry.getBrowserWorkspaceId(target.browserId), - ), - }, - }; -} - async function executeSnapshot( requestId: string, workspaceId: string | undefined, @@ -625,81 +488,6 @@ async function executeFill( return { requestId, ok: true, result: { command: "fill", browserId: target.browserId, ref } }; } -async function executeFocus( - requestId: string, - workspaceId: string | undefined, - browserId: string, - ref: string, - registry: BrowserRegistry, - snapshotEngine: BrowserSnapshotEngine, -): Promise { - const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); - if ("ok" in target) { - return target; - } - const result = await snapshotEngine.focus({ - browserId: target.browserId, - page: target.contents, - ref, - }); - if (!result.ok) { - return staleRefFailure(requestId, ref); - } - return { requestId, ok: true, result: { command: "focus", browserId: target.browserId, ref } }; -} - -async function executeClear( - requestId: string, - workspaceId: string | undefined, - browserId: string, - ref: string, - registry: BrowserRegistry, - snapshotEngine: BrowserSnapshotEngine, -): Promise { - const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); - if ("ok" in target) { - return target; - } - const result = await snapshotEngine.clear({ - browserId: target.browserId, - page: target.contents, - ref, - }); - if (!result.ok) { - return staleRefFailure(requestId, ref); - } - return { requestId, ok: true, result: { command: "clear", browserId: target.browserId, ref } }; -} - -async function executeCheck( - requestId: string, - workspaceId: string | undefined, - browserId: string, - ref: string, - checked: boolean, - registry: BrowserRegistry, - snapshotEngine: BrowserSnapshotEngine, -): Promise { - const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); - if ("ok" in target) { - return target; - } - const result = await snapshotEngine.check({ - browserId: target.browserId, - page: target.contents, - ref, - checked, - }); - if (!result.ok) { - return staleRefFailure(requestId, ref); - } - return { - requestId, - ok: true, - result: { command: "check", browserId: target.browserId, ref, checked }, - }; -} - async function executeSelect( requestId: string, workspaceId: string | undefined, @@ -808,84 +596,6 @@ async function executeLogs( }; } -async function executeStorage( - requestId: string, - workspaceId: string | undefined, - browserId: string, - registry: BrowserRegistry, -): Promise { - const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); - if ("ok" in target) { - return target; - } - const url = target.contents.getURL(); - const cookies = (await target.contents.getCookies?.(url)) ?? []; - const storage = parseStorageState(await target.contents.executeJavaScript(STORAGE_STATE_SCRIPT)); - return { - requestId, - ok: true, - result: { - command: "storage", - browserId: target.browserId, - url, - cookies, - localStorage: storage.localStorage, - sessionStorage: storage.sessionStorage, - }, - }; -} - -async function executeEnvironment( - requestId: string, - workspaceId: string | undefined, - browserId: string, - environment: { - viewport?: { width: number; height: number; deviceScaleFactor?: number }; - geolocation?: { latitude: number; longitude: number; accuracy?: number }; - }, - registry: BrowserRegistry, -): Promise { - const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); - if ("ok" in target) { - return target; - } - if (environment.viewport) { - await target.contents.sendDebugCommand?.("Emulation.setDeviceMetricsOverride", { - width: environment.viewport.width, - height: environment.viewport.height, - deviceScaleFactor: environment.viewport.deviceScaleFactor ?? 1, - mobile: false, - }); - } - if (environment.geolocation) { - const geolocation = { - latitude: environment.geolocation.latitude, - longitude: environment.geolocation.longitude, - accuracy: environment.geolocation.accuracy ?? 1, - }; - await target.contents.sendDebugCommand?.("Emulation.setGeolocationOverride", geolocation); - await target.contents.executeJavaScript(buildGeolocationShimScript(geolocation)); - } - const viewport = parseViewport(await target.contents.executeJavaScript(VIEWPORT_SCRIPT)); - return { - requestId, - ok: true, - result: { - command: "environment", - browserId: target.browserId, - viewport, - ...(environment.geolocation - ? { - geolocation: { - ...environment.geolocation, - accuracy: environment.geolocation.accuracy ?? 1, - }, - } - : {}), - }, - }; -} - function staleRefFailure(requestId: string, ref: string): FailurePayload { return fail( requestId, @@ -952,39 +662,6 @@ async function executeWait( return fail(requestId, "browser_unsupported", "browser_wait requires text or url"); } -async function executeSetBackground( - requestId: string, - workspaceId: string | undefined, - browserId: string, - color: string, - registry: BrowserRegistry, -): Promise { - const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); - if ("ok" in target) { - return target; - } - - await target.contents.executeJavaScript(buildSetBackgroundScript(color)); - return { - requestId, - ok: true, - result: { command: "set_background", browserId: target.browserId, color }, - }; -} - -function buildSetBackgroundScript(color: string): string { - return String.raw`(() => { - const color = ${JSON.stringify(color)}; - document.documentElement.style.background = color; - if (document.body) { - document.body.style.background = color; - document.body.style.backgroundColor = color; - document.body.style.minHeight = '100vh'; - } - return true; - })()`; -} - async function executeType( requestId: string, workspaceId: string | undefined, @@ -1092,8 +769,13 @@ async function executeScreenshot( requestId: string, workspaceId: string | undefined, browserId: string, + fullPage: boolean, registry: BrowserRegistry, ): Promise { + if (fullPage) { + return executeFullPageScreenshot(requestId, workspaceId, browserId, registry); + } + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); if ("ok" in target) { return target; @@ -1176,7 +858,7 @@ async function executeFullPageScreenshot( return target; } if (!target.contents.sendDebugCommand) { - return fail(requestId, "browser_unsupported", "browser_full_page_screenshot requires CDP"); + return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP"); } const metrics = await getCdpLayoutMetrics(target.contents); const width = metrics.contentWidth; @@ -1197,13 +879,13 @@ async function executeFullPageScreenshot( throw error; } if (!screenshot.data) { - return fail(requestId, "browser_unsupported", "browser_full_page_screenshot returned no data"); + return fail(requestId, "browser_unsupported", "browser_screenshot fullPage returned no data"); } return { requestId, ok: true, result: { - command: "full_page_screenshot", + command: "screenshot", browserId: target.browserId, mimeType: "image/png", dataBase64: screenshot.data, @@ -1213,68 +895,6 @@ async function executeFullPageScreenshot( }; } -async function executePdf( - requestId: string, - workspaceId: string | undefined, - browserId: string, - options: { landscape?: boolean; printBackground: boolean }, - registry: BrowserRegistry, -): Promise { - const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); - if ("ok" in target) { - return target; - } - if (!target.contents.printToPDF) { - return fail(requestId, "browser_unsupported", "browser_pdf requires PDF support"); - } - const pdf = await target.contents.printToPDF({ - printBackground: options.printBackground, - ...(options.landscape !== undefined ? { landscape: options.landscape } : {}), - }); - return { - requestId, - ok: true, - result: { - command: "pdf", - browserId: target.browserId, - mimeType: "application/pdf", - dataBase64: Buffer.from(pdf).toString("base64"), - }, - }; -} - -async function executeDownload( - requestId: string, - workspaceId: string | undefined, - browserId: string, - input: { url: string; fileName?: string }, - registry: BrowserRegistry, -): Promise { - const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); - if ("ok" in target) { - return target; - } - if (!isAllowedPageUrl(input.url)) { - return fail(requestId, "browser_denied", "Browser download only supports http and https URLs."); - } - if (!target.contents.downloadURL) { - return fail(requestId, "browser_unsupported", "browser_download requires download support"); - } - const download = await target.contents.downloadURL(input); - return { - requestId, - ok: true, - result: { - command: "download", - browserId: target.browserId, - url: input.url, - filePath: download.filePath, - ...(download.totalBytes !== undefined ? { totalBytes: download.totalBytes } : {}), - state: download.state, - }, - }; -} - function isAllowedPageUrl(value: string): boolean { try { return ALLOWED_PAGE_URL_PROTOCOLS.has(new URL(value).protocol); @@ -1408,36 +1028,6 @@ function parseNetworkEntries(value: unknown): BrowserAutomationNetworkLogEntry[] }); } -function parseStorageState(value: unknown): { - localStorage: BrowserAutomationStorageEntry[]; - sessionStorage: BrowserAutomationStorageEntry[]; -} { - const parsed = typeof value === "string" ? JSON.parse(value) : value; - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - return { localStorage: [], sessionStorage: [] }; - } - const record = parsed as Record; - return { - localStorage: parseStorageEntries(record.localStorage), - sessionStorage: parseStorageEntries(record.sessionStorage), - }; -} - -function parseStorageEntries(value: unknown): BrowserAutomationStorageEntry[] { - if (!Array.isArray(value)) { - return []; - } - return value.flatMap((entry): BrowserAutomationStorageEntry[] => { - if (!entry || typeof entry !== "object") { - return []; - } - const record = entry as Record; - const key = readString(record.key); - const itemValue = readString(record.value); - return key !== null && itemValue !== null ? [{ key, value: itemValue }] : []; - }); -} - function readString(value: unknown): string | null { return typeof value === "string" ? value : null; } @@ -1446,43 +1036,6 @@ function readNumber(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } -function parseViewport(value: unknown): { - width: number; - height: number; - deviceScaleFactor: number; -} { - const parsed = typeof value === "string" ? JSON.parse(value) : value; - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - return { width: 0, height: 0, deviceScaleFactor: 1 }; - } - const record = parsed as Record; - return { - width: readNumber(record.width) ?? 0, - height: readNumber(record.height) ?? 0, - deviceScaleFactor: readNumber(record.deviceScaleFactor) ?? 1, - }; -} - -function buildGeolocationShimScript(geolocation: { - latitude: number; - longitude: number; - accuracy: number; -}): string { - return String.raw`(() => { - const coords = ${JSON.stringify({ ...geolocation, altitude: null, altitudeAccuracy: null, heading: null, speed: null })}; - const position = { coords, timestamp: Date.now() }; - Object.defineProperty(navigator, 'geolocation', { - configurable: true, - value: { - getCurrentPosition(success) { setTimeout(() => success(position), 0); }, - watchPosition(success) { setTimeout(() => success(position), 0); return 1; }, - clearWatch() {}, - }, - }); - return true; - })()`; -} - const NETWORK_PERFORMANCE_SCRIPT = String.raw`(() => { const entries = performance.getEntriesByType('resource') .concat(performance.getEntriesByType('navigation')) @@ -1498,34 +1051,6 @@ const NETWORK_PERFORMANCE_SCRIPT = String.raw`(() => { return JSON.stringify(entries); })()`; -const STORAGE_STATE_SCRIPT = String.raw`(() => { - function entriesFor(storage) { - const entries = []; - for (let index = 0; index < storage.length; index += 1) { - const key = storage.key(index); - if (key !== null) entries.push({ key, value: storage.getItem(key) || '' }); - } - return entries; - } - function safeEntries(readStorage) { - try { - return entriesFor(readStorage()); - } catch { - return []; - } - } - return JSON.stringify({ - localStorage: safeEntries(() => window.localStorage), - sessionStorage: safeEntries(() => window.sessionStorage), - }); -})()`; - -const VIEWPORT_SCRIPT = String.raw`(() => JSON.stringify({ - width: window.innerWidth, - height: window.innerHeight, - deviceScaleFactor: window.devicePixelRatio || 1, -}))()`; - function resolveTabTarget(input: { requestId: string; workspaceId: string | undefined; diff --git a/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts b/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts index 22bd60936..eb0331326 100644 --- a/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts +++ b/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts @@ -44,7 +44,9 @@ describe("BrowserSnapshotEngine", () => { ok: false, reason: "stale_ref", }); - await expect(engine.focus({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({ + await expect( + engine.select({ browserId: "browser-1", page, ref: "@e1", value: "us" }), + ).resolves.toEqual({ ok: false, reason: "stale_ref", }); diff --git a/packages/desktop/src/features/browser-automation/snapshot-engine.ts b/packages/desktop/src/features/browser-automation/snapshot-engine.ts index 4d3c09745..8e0fe1b7b 100644 --- a/packages/desktop/src/features/browser-automation/snapshot-engine.ts +++ b/packages/desktop/src/features/browser-automation/snapshot-engine.ts @@ -108,31 +108,6 @@ export class BrowserSnapshotEngine { return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true }; } - async focus(input: { - browserId: string; - page: SnapshotPage; - ref: string; - }): Promise { - return this.runRefScript(input, (selector) => buildFocusScript(selector)); - } - - async clear(input: { - browserId: string; - page: SnapshotPage; - ref: string; - }): Promise { - return this.runRefScript(input, (selector) => buildClearScript(selector)); - } - - async check(input: { - browserId: string; - page: SnapshotPage; - ref: string; - checked: boolean; - }): Promise { - return this.runRefScript(input, (selector) => buildCheckScript(selector, input.checked)); - } - async select(input: { browserId: string; page: SnapshotPage; @@ -304,57 +279,6 @@ function buildKeypressScript(selector: string | undefined, key: string): string })()`; } -function buildFocusScript(selector: string): string { - return String.raw`(() => { - const element = document.querySelector(${JSON.stringify(selector)}); - if (!element) return false; - element.scrollIntoView?.({ block: 'center', inline: 'center' }); - element.focus?.(); - return document.activeElement === element; - })()`; -} - -function buildClearScript(selector: string): string { - return String.raw`(() => { - const element = document.querySelector(${JSON.stringify(selector)}); - if (!element) return false; - element.scrollIntoView?.({ block: 'center', inline: 'center' }); - element.focus?.(); - if ('value' in element) { - element.value = ''; - element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' })); - element.dispatchEvent(new Event('change', { bubbles: true })); - return true; - } - element.textContent = ''; - element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' })); - return true; - })()`; -} - -function buildCheckScript(selector: string, checked: boolean): string { - return String.raw`(() => { - const element = document.querySelector(${JSON.stringify(selector)}); - if (!element) return false; - element.scrollIntoView?.({ block: 'center', inline: 'center' }); - element.focus?.(); - const nextChecked = ${JSON.stringify(checked)}; - if ('checked' in element) { - element.checked = nextChecked; - element.dispatchEvent(new Event('input', { bubbles: true })); - element.dispatchEvent(new Event('change', { bubbles: true })); - return true; - } - if (element.getAttribute('role') === 'checkbox' || element.getAttribute('role') === 'radio') { - element.setAttribute('aria-checked', String(nextChecked)); - element.dispatchEvent(new Event('input', { bubbles: true })); - element.dispatchEvent(new Event('change', { bubbles: true })); - return true; - } - return false; - })()`; -} - function buildSelectScript(selector: string, value: string): string { return String.raw`(() => { const element = document.querySelector(${JSON.stringify(selector)}); diff --git a/packages/protocol/src/browser-automation/rpc-schemas.test.ts b/packages/protocol/src/browser-automation/rpc-schemas.test.ts index 2f5d0c7f1..3bf53e19e 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.test.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.test.ts @@ -73,28 +73,12 @@ const commandParseCases = [ { name: "screenshot", command: { command: "screenshot", args: { browserId: BROWSER_ID } }, - expected: { command: "screenshot", args: { browserId: BROWSER_ID } }, + expected: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: false } }, }, { name: "full page screenshot", - command: { command: "full_page_screenshot", args: { browserId: BROWSER_ID } }, - expected: { command: "full_page_screenshot", args: { browserId: BROWSER_ID } }, - }, - { - name: "pdf", - command: { command: "pdf", args: { browserId: BROWSER_ID } }, - expected: { command: "pdf", args: { browserId: BROWSER_ID, printBackground: true } }, - }, - { - name: "download", - command: { - command: "download", - args: { browserId: BROWSER_ID, url: "https://example.com/file.txt" }, - }, - expected: { - command: "download", - args: { browserId: BROWSER_ID, url: "https://example.com/file.txt" }, - }, + command: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: true } }, + expected: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: true } }, }, { name: "upload", @@ -107,21 +91,6 @@ const commandParseCases = [ args: { browserId: BROWSER_ID, ref: "@e1", filePaths: ["/tmp/file.txt"] }, }, }, - { - name: "focus", - command: { command: "focus", args: { browserId: BROWSER_ID, ref: "@e1" } }, - expected: { command: "focus", args: { browserId: BROWSER_ID, ref: "@e1" } }, - }, - { - name: "clear", - command: { command: "clear", args: { browserId: BROWSER_ID, ref: "@e1" } }, - expected: { command: "clear", args: { browserId: BROWSER_ID, ref: "@e1" } }, - }, - { - name: "check", - command: { command: "check", args: { browserId: BROWSER_ID, ref: "@e2" } }, - expected: { command: "check", args: { browserId: BROWSER_ID, ref: "@e2", checked: true } }, - }, { name: "select", command: { @@ -154,61 +123,9 @@ const commandParseCases = [ command: { command: "logs", args: { browserId: BROWSER_ID } }, expected: { command: "logs", args: { browserId: BROWSER_ID, maxEntries: 50 } }, }, - { - name: "storage", - command: { command: "storage", args: { browserId: BROWSER_ID } }, - expected: { command: "storage", args: { browserId: BROWSER_ID } }, - }, - { - name: "environment", - command: { - command: "environment", - args: { - browserId: BROWSER_ID, - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, - expected: { - command: "environment", - args: { - browserId: BROWSER_ID, - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, - }, - { - name: "set background", - command: { command: "set_background", args: { browserId: BROWSER_ID, color: "red" } }, - expected: { command: "set_background", args: { browserId: BROWSER_ID, color: "red" } }, - }, ] as const; const resultParseCases = [ - { - name: "page info", - result: { - command: "page_info", - tab: { - browserId: BROWSER_ID, - workspaceId: "workspace-1", - url: "https://example.com", - title: "Example", - }, - }, - expected: { - command: "page_info", - tab: { - browserId: BROWSER_ID, - workspaceId: "workspace-1", - url: "https://example.com", - title: "Example", - isActive: false, - isLoading: false, - }, - }, - }, { name: "snapshot", result: { @@ -313,7 +230,7 @@ const resultParseCases = [ { name: "full page screenshot", result: { - command: "full_page_screenshot", + command: "screenshot", browserId: BROWSER_ID, mimeType: "image/png", dataBase64: "iVBORw0KGgo=", @@ -321,7 +238,7 @@ const resultParseCases = [ height: 1200, }, expected: { - command: "full_page_screenshot", + command: "screenshot", browserId: BROWSER_ID, mimeType: "image/png", dataBase64: "iVBORw0KGgo=", @@ -329,40 +246,6 @@ const resultParseCases = [ height: 1200, }, }, - { - name: "pdf", - result: { - command: "pdf", - browserId: BROWSER_ID, - mimeType: "application/pdf", - dataBase64: "JVBERg==", - }, - expected: { - command: "pdf", - browserId: BROWSER_ID, - mimeType: "application/pdf", - dataBase64: "JVBERg==", - }, - }, - { - name: "download", - result: { - command: "download", - browserId: BROWSER_ID, - url: "https://example.com/file.txt", - filePath: "/tmp/file.txt", - totalBytes: 5, - state: "completed", - }, - expected: { - command: "download", - browserId: BROWSER_ID, - url: "https://example.com/file.txt", - filePath: "/tmp/file.txt", - totalBytes: 5, - state: "completed", - }, - }, { name: "upload", result: { @@ -378,21 +261,6 @@ const resultParseCases = [ filePaths: ["/tmp/file.txt"], }, }, - { - name: "focus", - result: { command: "focus", browserId: BROWSER_ID, ref: "@e1" }, - expected: { command: "focus", browserId: BROWSER_ID, ref: "@e1" }, - }, - { - name: "clear", - result: { command: "clear", browserId: BROWSER_ID, ref: "@e1" }, - expected: { command: "clear", browserId: BROWSER_ID, ref: "@e1" }, - }, - { - name: "check", - result: { command: "check", browserId: BROWSER_ID, ref: "@e2", checked: false }, - expected: { command: "check", browserId: BROWSER_ID, ref: "@e2", checked: false }, - }, { name: "select", result: { command: "select", browserId: BROWSER_ID, ref: "@e3", value: "us" }, @@ -437,45 +305,6 @@ const resultParseCases = [ ], }, }, - { - name: "storage", - result: { - command: "storage", - browserId: BROWSER_ID, - url: "https://example.com", - cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }, - expected: { - command: "storage", - browserId: BROWSER_ID, - url: "https://example.com", - cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }, - }, - { - name: "environment", - result: { - command: "environment", - browserId: BROWSER_ID, - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - expected: { - command: "environment", - browserId: BROWSER_ID, - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, - { - name: "set background", - result: { command: "set_background", browserId: BROWSER_ID, color: "red" }, - expected: { command: "set_background", browserId: BROWSER_ID, color: "red" }, - }, ] as const; describe("browser automation execute RPC schemas", () => { @@ -526,9 +355,9 @@ describe("browser automation execute RPC schemas", () => { test("tab commands reject hallucinated browser ids", () => { const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ type: "browser.automation.execute.request", - requestId: "req-page-info", + requestId: "req-snapshot", workspaceId: "workspace-1", - command: { command: "page_info", args: { browserId: "default" } }, + command: { command: "snapshot", args: { browserId: "default" } }, }); expect(parsed).toMatchObject({ @@ -540,13 +369,13 @@ describe("browser automation execute RPC schemas", () => { test("tab commands parse browser ids produced by the fallback generator", () => { const parsed = BrowserAutomationExecuteRequestSchema.parse({ type: "browser.automation.execute.request", - requestId: "req-page-info", + requestId: "req-snapshot", workspaceId: "workspace-1", - command: { command: "page_info", args: { browserId: FALLBACK_BROWSER_ID } }, + command: { command: "snapshot", args: { browserId: FALLBACK_BROWSER_ID } }, }); expect(parsed.command).toEqual({ - command: "page_info", + command: "snapshot", args: { browserId: FALLBACK_BROWSER_ID }, }); }); @@ -658,22 +487,6 @@ describe("browser automation execute RPC schemas", () => { }); }); - test("download rejects non-http URLs at the protocol boundary", () => { - const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ - type: "browser.automation.execute.request", - requestId: "req-download", - command: { - command: "download", - args: { browserId: BROWSER_ID, url: "file:///tmp/secret.txt" }, - }, - }); - - expect(parsed).toMatchObject({ - success: false, - error: { issues: [expect.objectContaining({ message: "URL must use http or https" })] }, - }); - }); - test("new tab responses declare the generated browser id shape", () => { const parsed = BrowserAutomationExecuteResponseSchema.parse({ type: "browser.automation.execute.response", @@ -705,16 +518,15 @@ describe("browser automation execute RPC schemas", () => { const parsed = BrowserAutomationExecuteResponseSchema.safeParse({ type: "browser.automation.execute.response", payload: { - requestId: "req-page-info", + requestId: "req-snapshot", ok: true, result: { - command: "page_info", - tab: { - browserId: "default", - workspaceId: "workspace-1", - url: "https://example.com", - title: "Example", - }, + command: "snapshot", + browserId: "default", + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + elements: [], }, }, }); diff --git a/packages/protocol/src/browser-automation/rpc-schemas.ts b/packages/protocol/src/browser-automation/rpc-schemas.ts index b47b70d10..593b15c13 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.ts @@ -55,11 +55,6 @@ export const BrowserAutomationNewTabCommandSchema = z.object({ .default({}), }); -export const BrowserAutomationPageInfoCommandSchema = z.object({ - command: z.literal("page_info"), - args: BrowserAutomationTabTargetSchema, -}); - export const BrowserAutomationSnapshotCommandSchema = z.object({ command: z.literal("snapshot"), args: BrowserAutomationTabTargetSchema, @@ -131,27 +126,8 @@ export const BrowserAutomationReloadCommandSchema = z.object({ export const BrowserAutomationScreenshotCommandSchema = z.object({ command: z.literal("screenshot"), - args: BrowserAutomationTabTargetSchema, -}); - -export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({ - command: z.literal("full_page_screenshot"), - args: BrowserAutomationTabTargetSchema, -}); - -export const BrowserAutomationPdfCommandSchema = z.object({ - command: z.literal("pdf"), args: BrowserAutomationTabTargetSchema.extend({ - landscape: z.boolean().optional(), - printBackground: z.boolean().default(true), - }), -}); - -export const BrowserAutomationDownloadCommandSchema = z.object({ - command: z.literal("download"), - args: BrowserAutomationTabTargetSchema.extend({ - url: BrowserAutomationHttpUrlSchema, - fileName: z.string().min(1).optional(), + fullPage: z.boolean().default(false), }), }); @@ -163,28 +139,6 @@ export const BrowserAutomationUploadCommandSchema = z.object({ }), }); -export const BrowserAutomationFocusCommandSchema = z.object({ - command: z.literal("focus"), - args: BrowserAutomationTabTargetSchema.extend({ - ref: BrowserAutomationRefSchema, - }), -}); - -export const BrowserAutomationClearCommandSchema = z.object({ - command: z.literal("clear"), - args: BrowserAutomationTabTargetSchema.extend({ - ref: BrowserAutomationRefSchema, - }), -}); - -export const BrowserAutomationCheckCommandSchema = z.object({ - command: z.literal("check"), - args: BrowserAutomationTabTargetSchema.extend({ - ref: BrowserAutomationRefSchema, - checked: z.boolean().default(true), - }), -}); - export const BrowserAutomationSelectCommandSchema = z.object({ command: z.literal("select"), args: BrowserAutomationTabTargetSchema.extend({ @@ -215,42 +169,9 @@ export const BrowserAutomationLogsCommandSchema = z.object({ }), }); -export const BrowserAutomationStorageCommandSchema = z.object({ - command: z.literal("storage"), - args: BrowserAutomationTabTargetSchema, -}); - -const BrowserAutomationViewportInputSchema = z.object({ - width: z.number().int().positive(), - height: z.number().int().positive(), - deviceScaleFactor: z.number().positive().optional(), -}); - -const BrowserAutomationGeolocationInputSchema = z.object({ - latitude: z.number().min(-90).max(90), - longitude: z.number().min(-180).max(180), - accuracy: z.number().positive().optional(), -}); - -export const BrowserAutomationEnvironmentCommandSchema = z.object({ - command: z.literal("environment"), - args: BrowserAutomationTabTargetSchema.extend({ - viewport: BrowserAutomationViewportInputSchema.optional(), - geolocation: BrowserAutomationGeolocationInputSchema.optional(), - }), -}); - -export const BrowserAutomationSetBackgroundCommandSchema = z.object({ - command: z.literal("set_background"), - args: BrowserAutomationTabTargetSchema.extend({ - color: z.string().min(1), - }), -}); - export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [ BrowserAutomationListTabsCommandSchema, BrowserAutomationNewTabCommandSchema, - BrowserAutomationPageInfoCommandSchema, BrowserAutomationSnapshotCommandSchema, BrowserAutomationClickCommandSchema, BrowserAutomationFillCommandSchema, @@ -262,20 +183,11 @@ export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [ BrowserAutomationForwardCommandSchema, BrowserAutomationReloadCommandSchema, BrowserAutomationScreenshotCommandSchema, - BrowserAutomationFullPageScreenshotCommandSchema, - BrowserAutomationPdfCommandSchema, - BrowserAutomationDownloadCommandSchema, BrowserAutomationUploadCommandSchema, - BrowserAutomationFocusCommandSchema, - BrowserAutomationClearCommandSchema, - BrowserAutomationCheckCommandSchema, BrowserAutomationSelectCommandSchema, BrowserAutomationHoverCommandSchema, BrowserAutomationDragCommandSchema, BrowserAutomationLogsCommandSchema, - BrowserAutomationStorageCommandSchema, - BrowserAutomationEnvironmentCommandSchema, - BrowserAutomationSetBackgroundCommandSchema, ]); export const BrowserAutomationTabInfoSchema = z.object({ @@ -301,11 +213,6 @@ export const BrowserAutomationNewTabResultSchema = z.object({ url: z.string().min(1), }); -export const BrowserAutomationPageInfoResultSchema = z.object({ - command: z.literal("page_info"), - tab: BrowserAutomationTabInfoSchema, -}); - export const BrowserAutomationSnapshotElementSchema = z.object({ ref: z.string().regex(/^@e\d+$/), role: z.string(), @@ -385,31 +292,6 @@ export const BrowserAutomationScreenshotResultSchema = z.object({ height: z.number().int().nonnegative(), }); -export const BrowserAutomationFullPageScreenshotResultSchema = z.object({ - command: z.literal("full_page_screenshot"), - browserId: BrowserAutomationBrowserIdSchema, - mimeType: z.literal("image/png"), - dataBase64: z.string().min(1), - width: z.number().int().nonnegative(), - height: z.number().int().nonnegative(), -}); - -export const BrowserAutomationPdfResultSchema = z.object({ - command: z.literal("pdf"), - browserId: BrowserAutomationBrowserIdSchema, - mimeType: z.literal("application/pdf"), - dataBase64: z.string().min(1), -}); - -export const BrowserAutomationDownloadResultSchema = z.object({ - command: z.literal("download"), - browserId: BrowserAutomationBrowserIdSchema, - url: z.string().min(1), - filePath: z.string().min(1), - totalBytes: z.number().int().nonnegative().optional(), - state: z.string().min(1), -}); - export const BrowserAutomationUploadResultSchema = z.object({ command: z.literal("upload"), browserId: BrowserAutomationBrowserIdSchema, @@ -417,25 +299,6 @@ export const BrowserAutomationUploadResultSchema = z.object({ filePaths: z.array(z.string().min(1)).min(1), }); -export const BrowserAutomationFocusResultSchema = z.object({ - command: z.literal("focus"), - browserId: BrowserAutomationBrowserIdSchema, - ref: BrowserAutomationRefSchema, -}); - -export const BrowserAutomationClearResultSchema = z.object({ - command: z.literal("clear"), - browserId: BrowserAutomationBrowserIdSchema, - ref: BrowserAutomationRefSchema, -}); - -export const BrowserAutomationCheckResultSchema = z.object({ - command: z.literal("check"), - browserId: BrowserAutomationBrowserIdSchema, - ref: BrowserAutomationRefSchema, - checked: z.boolean(), -}); - export const BrowserAutomationSelectResultSchema = z.object({ command: z.literal("select"), browserId: BrowserAutomationBrowserIdSchema, @@ -481,59 +344,9 @@ export const BrowserAutomationLogsResultSchema = z.object({ network: z.array(BrowserAutomationNetworkLogEntrySchema), }); -export const BrowserAutomationCookieEntrySchema = z.object({ - name: z.string(), - value: z.string(), - domain: z.string().optional(), - path: z.string().optional(), - secure: z.boolean().optional(), - httpOnly: z.boolean().optional(), - expirationDate: z.number().optional(), -}); - -export const BrowserAutomationStorageEntrySchema = z.object({ - key: z.string(), - value: z.string(), -}); - -export const BrowserAutomationStorageResultSchema = z.object({ - command: z.literal("storage"), - browserId: BrowserAutomationBrowserIdSchema, - url: z.string(), - cookies: z.array(BrowserAutomationCookieEntrySchema), - localStorage: z.array(BrowserAutomationStorageEntrySchema), - sessionStorage: z.array(BrowserAutomationStorageEntrySchema), -}); - -export const BrowserAutomationViewportResultSchema = z.object({ - width: z.number().int().nonnegative(), - height: z.number().int().nonnegative(), - deviceScaleFactor: z.number().positive(), -}); - -export const BrowserAutomationGeolocationResultSchema = z.object({ - latitude: z.number(), - longitude: z.number(), - accuracy: z.number(), -}); - -export const BrowserAutomationEnvironmentResultSchema = z.object({ - command: z.literal("environment"), - browserId: BrowserAutomationBrowserIdSchema, - viewport: BrowserAutomationViewportResultSchema, - geolocation: BrowserAutomationGeolocationResultSchema.optional(), -}); - -export const BrowserAutomationSetBackgroundResultSchema = z.object({ - command: z.literal("set_background"), - browserId: BrowserAutomationBrowserIdSchema, - color: z.string().min(1), -}); - export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [ BrowserAutomationListTabsResultSchema, BrowserAutomationNewTabResultSchema, - BrowserAutomationPageInfoResultSchema, BrowserAutomationSnapshotResultSchema, BrowserAutomationClickResultSchema, BrowserAutomationFillResultSchema, @@ -545,20 +358,11 @@ export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [ BrowserAutomationForwardResultSchema, BrowserAutomationReloadResultSchema, BrowserAutomationScreenshotResultSchema, - BrowserAutomationFullPageScreenshotResultSchema, - BrowserAutomationPdfResultSchema, - BrowserAutomationDownloadResultSchema, BrowserAutomationUploadResultSchema, - BrowserAutomationFocusResultSchema, - BrowserAutomationClearResultSchema, - BrowserAutomationCheckResultSchema, BrowserAutomationSelectResultSchema, BrowserAutomationHoverResultSchema, BrowserAutomationDragResultSchema, BrowserAutomationLogsResultSchema, - BrowserAutomationStorageResultSchema, - BrowserAutomationEnvironmentResultSchema, - BrowserAutomationSetBackgroundResultSchema, ]); export const BrowserAutomationErrorSchema = z.object({ @@ -603,8 +407,6 @@ export type BrowserAutomationConsoleLogEntry = z.infer< export type BrowserAutomationNetworkLogEntry = z.infer< typeof BrowserAutomationNetworkLogEntrySchema >; -export type BrowserAutomationCookieEntry = z.infer; -export type BrowserAutomationStorageEntry = z.infer; export type BrowserAutomationExecuteRequest = z.infer; export type BrowserAutomationExecuteResponse = z.infer< typeof BrowserAutomationExecuteResponseSchema diff --git a/packages/protocol/src/messages.browser-automation.test.ts b/packages/protocol/src/messages.browser-automation.test.ts index d864f8f0a..5b3a8a721 100644 --- a/packages/protocol/src/messages.browser-automation.test.ts +++ b/packages/protocol/src/messages.browser-automation.test.ts @@ -41,7 +41,7 @@ describe("browser automation protocol integration", () => { const parsed = SessionOutboundMessageSchema.parse({ type: "browser.automation.execute.request", requestId: "req-1", - command: { command: "page_info", args: { browserId } }, + command: { command: "snapshot", args: { browserId } }, }); expect(parsed.type).toBe("browser.automation.execute.request"); diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index ae972129a..00c96da61 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -753,7 +753,7 @@ describe("browser MCP tools", () => { const response = await tool.handler({}); - expect(lookupTool(server, "browser_page_info")).not.toBeUndefined(); + expect(lookupTool(server, "browser_snapshot")).not.toBeUndefined(); expect(execute).toHaveBeenCalledWith({ agentId: "agent-1", cwd: REPO_CWD, diff --git a/packages/server/src/server/browser-tools/broker.test.ts b/packages/server/src/server/browser-tools/broker.test.ts index ea6856baf..dc5a84f07 100644 --- a/packages/server/src/server/browser-tools/broker.test.ts +++ b/packages/server/src/server/browser-tools/broker.test.ts @@ -45,8 +45,8 @@ function createBroker(options: { enabled: boolean; timeoutMs?: number }): Browse }); } -function pageInfoCommand(): BrowserAutomationCommand { - return { command: "page_info", args: { browserId: BROWSER_ID } }; +function snapshotCommand(): BrowserAutomationCommand { + return { command: "snapshot", args: { browserId: BROWSER_ID } }; } describe("BrowserToolsBroker", () => { @@ -57,7 +57,7 @@ describe("BrowserToolsBroker", () => { test("disabled returns browser_disabled", async () => { const broker = createBroker({ enabled: false }); - await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({ + await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({ requestId: "req-1", ok: false, error: { @@ -71,7 +71,7 @@ describe("BrowserToolsBroker", () => { test("no capable desktop returns browser_no_desktop", async () => { const broker = createBroker({ enabled: true }); - await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({ + await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({ requestId: "req-1", ok: false, error: { diff --git a/packages/server/src/server/browser-tools/tools.test.ts b/packages/server/src/server/browser-tools/tools.test.ts index 7d697d361..8c724d82b 100644 --- a/packages/server/src/server/browser-tools/tools.test.ts +++ b/packages/server/src/server/browser-tools/tools.test.ts @@ -68,6 +68,10 @@ class BrowserToolHarness { return this.get(name).handler(parsed, {}); } + public toolNames(): string[] { + return Array.from(this.tools.keys()); + } + private get(name: string): RegisteredTool { const tool = this.tools.get(name); if (!tool) { @@ -150,18 +154,6 @@ function screenshotPayload(): Extract } const routedToolCases = [ - { - name: "set background", - toolName: "browser_set_background", - input: { browserId: BROWSER_ID, color: "red" }, - command: { command: "set_background", args: { browserId: BROWSER_ID, color: "red" } }, - payload: { - requestId: "req-set-background", - ok: true, - result: { command: "set_background", browserId: BROWSER_ID, color: "red" }, - }, - content: [{ type: "text", text: "Set browser page background to red." }], - }, { name: "click", toolName: "browser_click", @@ -226,7 +218,7 @@ const routedToolCases = [ name: "screenshot", toolName: "browser_screenshot", input: { browserId: BROWSER_ID }, - command: { command: "screenshot", args: { browserId: BROWSER_ID } }, + command: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: false } }, payload: screenshotPayload(), content: [ { type: "text", text: "Captured browser screenshot (800x600)." }, @@ -257,65 +249,16 @@ const routedToolCases = [ }, content: [{ type: "text", text: "Read 1 console log and 1 network entry." }], }, - { - name: "storage", - toolName: "browser_storage", - input: { browserId: BROWSER_ID }, - command: { command: "storage", args: { browserId: BROWSER_ID } }, - payload: { - requestId: "req-storage", - ok: true, - result: { - command: "storage", - browserId: BROWSER_ID, - url: "https://example.com", - cookies: [{ name: "theme", value: "dark" }], - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }, - }, - content: [ - { type: "text", text: "Read 1 cookie, 1 localStorage entry, and 1 sessionStorage entry." }, - ], - }, - { - name: "environment", - toolName: "browser_environment", - input: { - browserId: BROWSER_ID, - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - command: { - command: "environment", - args: { - browserId: BROWSER_ID, - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, - payload: { - requestId: "req-environment", - ok: true, - result: { - command: "environment", - browserId: BROWSER_ID, - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, - content: [{ type: "text", text: "Browser environment viewport is 390x844." }], - }, { name: "full page screenshot", - toolName: "browser_full_page_screenshot", - input: { browserId: BROWSER_ID }, - command: { command: "full_page_screenshot", args: { browserId: BROWSER_ID } }, + toolName: "browser_screenshot", + input: { browserId: BROWSER_ID, fullPage: true }, + command: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: true } }, payload: { requestId: "req-full-page", ok: true, result: { - command: "full_page_screenshot", + command: "screenshot", browserId: BROWSER_ID, mimeType: "image/png", dataBase64: "iVBORw0KGgo=", @@ -324,56 +267,10 @@ const routedToolCases = [ }, }, content: [ - { type: "text", text: "Captured full-page browser screenshot (390x1200)." }, + { type: "text", text: "Captured browser screenshot (390x1200)." }, { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }, ], }, - { - name: "pdf", - toolName: "browser_pdf", - input: { browserId: BROWSER_ID, landscape: true }, - command: { - command: "pdf", - args: { browserId: BROWSER_ID, landscape: true, printBackground: true }, - }, - payload: { - requestId: "req-pdf", - ok: true, - result: { - command: "pdf", - browserId: BROWSER_ID, - mimeType: "application/pdf", - dataBase64: "JVBERg==", - }, - }, - content: [{ type: "text", text: "Exported browser page PDF." }], - }, - { - name: "download", - toolName: "browser_download", - input: { browserId: BROWSER_ID, url: "https://example.com/file.txt", fileName: "file.txt" }, - command: { - command: "download", - args: { - browserId: BROWSER_ID, - url: "https://example.com/file.txt", - fileName: "file.txt", - }, - }, - payload: { - requestId: "req-download", - ok: true, - result: { - command: "download", - browserId: BROWSER_ID, - url: "https://example.com/file.txt", - filePath: "/tmp/file.txt", - totalBytes: 5, - state: "completed", - }, - }, - content: [{ type: "text", text: "Downloaded browser file to /tmp/file.txt." }], - }, { name: "upload", toolName: "browser_upload", @@ -394,42 +291,6 @@ const routedToolCases = [ }, content: [{ type: "text", text: "Uploaded 1 file to browser element @e1." }], }, - { - name: "focus", - toolName: "browser_focus", - input: { browserId: BROWSER_ID, ref: "@e1" }, - command: { command: "focus", args: { browserId: BROWSER_ID, ref: "@e1" } }, - payload: { - requestId: "req-focus", - ok: true, - result: { command: "focus", browserId: BROWSER_ID, ref: "@e1" }, - }, - content: [{ type: "text", text: "Focused browser element @e1." }], - }, - { - name: "clear", - toolName: "browser_clear", - input: { browserId: BROWSER_ID, ref: "@e1" }, - command: { command: "clear", args: { browserId: BROWSER_ID, ref: "@e1" } }, - payload: { - requestId: "req-clear", - ok: true, - result: { command: "clear", browserId: BROWSER_ID, ref: "@e1" }, - }, - content: [{ type: "text", text: "Cleared browser element @e1." }], - }, - { - name: "check", - toolName: "browser_check", - input: { browserId: BROWSER_ID, ref: "@e2", checked: false }, - command: { command: "check", args: { browserId: BROWSER_ID, ref: "@e2", checked: false } }, - payload: { - requestId: "req-check", - ok: true, - result: { command: "check", browserId: BROWSER_ID, ref: "@e2", checked: false }, - }, - content: [{ type: "text", text: "Unchecked browser element @e2." }], - }, { name: "select", toolName: "browser_select", @@ -502,7 +363,7 @@ const brokerErrorCases = [ }, { name: "typed timeout errors", - toolName: "browser_page_info", + toolName: "browser_snapshot", input: { browserId: BROWSER_ID }, payload: { requestId: "req-timeout", @@ -562,6 +423,31 @@ const brokerErrorCases = [ }>; describe("registerBrowserTools", () => { + test("registers the kept browser automation tools only", () => { + const harness = new BrowserToolHarness(); + + expect(harness.toolNames()).toEqual([ + "browser_list_tabs", + "browser_new_tab", + "browser_snapshot", + "browser_click", + "browser_fill", + "browser_wait", + "browser_type", + "browser_keypress", + "browser_navigate", + "browser_back", + "browser_forward", + "browser_reload", + "browser_screenshot", + "browser_upload", + "browser_hover", + "browser_select", + "browser_drag", + "browser_logs", + ]); + }); + test("list tabs sends workspace in the request envelope", async () => { const harness = new BrowserToolHarness(); @@ -612,12 +498,6 @@ describe("registerBrowserTools", () => { input: { browserId: BROWSER_ID, url: "localhost:3000" }, expected: { browserId: BROWSER_ID, url: "http://localhost:3000" }, }, - { - name: "download accepts an IP host without a scheme as http", - toolName: "browser_download", - input: { browserId: BROWSER_ID, url: "127.0.0.1:8081/admin" }, - expected: { browserId: BROWSER_ID, url: "http://127.0.0.1:8081/admin" }, - }, { name: "new tab accepts a domain path without a scheme as http", toolName: "browser_new_tab", @@ -662,11 +542,6 @@ describe("registerBrowserTools", () => { toolName: "browser_navigate", input: { browserId: BROWSER_ID, url: "file:///tmp/index.html" }, }, - { - name: "download rejects file URLs", - toolName: "browser_download", - input: { browserId: BROWSER_ID, url: "file:///tmp/archive.zip" }, - }, { name: "new tab rejects file URLs", toolName: "browser_new_tab", @@ -905,38 +780,16 @@ describe("registerBrowserTools", () => { test("tab tools keep empty context when there is no caller agent", async () => { const harness = new BrowserToolHarness(null, null); - harness.broker.setResponse({ - requestId: "req-page-info", - ok: true, - result: { - command: "page_info", - tab: { - browserId: BROWSER_ID, - url: "https://example.com", - title: "Example", - isActive: false, - isLoading: false, - }, - }, - }); + harness.broker.setResponse(snapshotPayload()); - const response = await harness.execute("browser_page_info", { browserId: BROWSER_ID }); + const response = await harness.execute("browser_snapshot", { browserId: BROWSER_ID }); expect(harness.broker.calls).toEqual([ - { command: { command: "page_info", args: { browserId: BROWSER_ID } } }, + { command: { command: "snapshot", args: { browserId: BROWSER_ID } } }, ]); expect(response.structuredContent).toEqual({ ok: true, - result: { - command: "page_info", - tab: { - browserId: BROWSER_ID, - url: "https://example.com", - title: "Example", - isActive: false, - isLoading: false, - }, - }, + result: snapshotPayload().result, context: { browserId: BROWSER_ID }, }); }); diff --git a/packages/server/src/server/browser-tools/tools.ts b/packages/server/src/server/browser-tools/tools.ts index e1f6423c1..2fbc664c2 100644 --- a/packages/server/src/server/browser-tools/tools.ts +++ b/packages/server/src/server/browser-tools/tools.ts @@ -140,35 +140,6 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void }, ); - options.registerTool( - "browser_page_info", - { - title: "Get browser page info", - description: - "Get page info for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.", - inputSchema: { - browserId: BrowserAutomationBrowserIdSchema, - }, - outputSchema: BrowserToolOutputSchema, - }, - async ({ browserId }) => { - const context = resolveBrowserToolContext(options); - const payload = await options.broker.execute({ - agentId: context.agentId, - cwd: context.cwd, - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - - command: { - command: "page_info", - args: { - browserId, - }, - }, - }); - return browserToolResult({ payload, context: { ...context, browserId } }); - }, - ); - options.registerTool( "browser_snapshot", { @@ -442,11 +413,14 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Capture browser screenshot", description: - "Capture a PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.", - inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, + "Capture a PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs. Set fullPage to true to capture the full page.", + inputSchema: { + browserId: BrowserAutomationBrowserIdSchema, + fullPage: z.boolean().default(false), + }, outputSchema: BrowserToolOutputSchema, }, - async ({ browserId }) => { + async ({ browserId, fullPage }) => { const context = resolveBrowserToolContext(options); const payload = await options.broker.execute({ agentId: context.agentId, @@ -457,99 +431,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void command: "screenshot", args: { browserId, - }, - }, - }); - return browserToolResult({ payload, context: { ...context, browserId } }); - }, - ); - - options.registerTool( - "browser_full_page_screenshot", - { - title: "Capture full-page browser screenshot", - description: - "Capture a full-page PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.", - inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, - outputSchema: BrowserToolOutputSchema, - }, - async ({ browserId }) => { - const context = resolveBrowserToolContext(options); - const payload = await options.broker.execute({ - agentId: context.agentId, - cwd: context.cwd, - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - - command: { - command: "full_page_screenshot", - args: { - browserId, - }, - }, - }); - return browserToolResult({ payload, context: { ...context, browserId } }); - }, - ); - - options.registerTool( - "browser_pdf", - { - title: "Export browser page PDF", - description: - "Export a Paseo desktop browser tab as a PDF. Use browserId from browser_new_tab or browser_list_tabs.", - inputSchema: { - browserId: BrowserAutomationBrowserIdSchema, - landscape: z.boolean().optional(), - printBackground: z.boolean().default(true), - }, - outputSchema: BrowserToolOutputSchema, - }, - async ({ browserId, landscape, printBackground }) => { - const context = resolveBrowserToolContext(options); - const payload = await options.broker.execute({ - agentId: context.agentId, - cwd: context.cwd, - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - - command: { - command: "pdf", - args: { - browserId, - ...(landscape !== undefined ? { landscape } : {}), - printBackground: printBackground ?? true, - }, - }, - }); - return browserToolResult({ payload, context: { ...context, browserId } }); - }, - ); - - options.registerTool( - "browser_download", - { - title: "Download file in browser", - description: - "Download a URL through a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; pass an http(s) URL or a scheme-less host URL, which is treated as http.", - inputSchema: { - url: BrowserHttpUrlInputSchema, - fileName: z.string().min(1).optional(), - browserId: BrowserAutomationBrowserIdSchema, - }, - outputSchema: BrowserToolOutputSchema, - }, - async ({ url, fileName, browserId }) => { - const context = resolveBrowserToolContext(options); - const payload = await options.broker.execute({ - agentId: context.agentId, - cwd: context.cwd, - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - - command: { - command: "download", - args: { - browserId, - url, - ...(fileName ? { fileName } : {}), + fullPage: fullPage ?? false, }, }, }); @@ -591,20 +473,6 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void ); for (const toolConfig of [ - { - name: "browser_focus", - command: "focus", - title: "Focus browser element", - description: - "Focus an element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", - }, - { - name: "browser_clear", - command: "clear", - title: "Clear browser element", - description: - "Clear an input-like element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", - }, { name: "browser_hover", command: "hover", @@ -641,39 +509,6 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void ); } - options.registerTool( - "browser_check", - { - title: "Check browser control", - description: - "Set a checkbox or radio control in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", - inputSchema: { - ref: BrowserRefInputSchema, - checked: z.boolean().default(true), - browserId: BrowserAutomationBrowserIdSchema, - }, - outputSchema: BrowserToolOutputSchema, - }, - async ({ ref, checked, browserId }) => { - const context = resolveBrowserToolContext(options); - const payload = await options.broker.execute({ - agentId: context.agentId, - cwd: context.cwd, - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - - command: { - command: "check", - args: { - browserId, - ref, - checked, - }, - }, - }); - return browserToolResult({ payload, context: { ...context, browserId } }); - }, - ); - options.registerTool( "browser_select", { @@ -770,109 +605,6 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void return browserToolResult({ payload, context: { ...context, browserId } }); }, ); - - options.registerTool( - "browser_storage", - { - title: "Read browser storage", - description: - "Read cookies plus localStorage and sessionStorage for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.", - inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, - outputSchema: BrowserToolOutputSchema, - }, - async ({ browserId }) => { - const context = resolveBrowserToolContext(options); - const payload = await options.broker.execute({ - agentId: context.agentId, - cwd: context.cwd, - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - - command: { - command: "storage", - args: { - browserId, - }, - }, - }); - return browserToolResult({ payload, context: { ...context, browserId } }); - }, - ); - - options.registerTool( - "browser_environment", - { - title: "Set/read browser environment", - description: - "Set or read viewport and geolocation for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.", - inputSchema: { - viewport: z - .object({ - width: z.number().int().positive(), - height: z.number().int().positive(), - deviceScaleFactor: z.number().positive().optional(), - }) - .optional(), - geolocation: z - .object({ - latitude: z.number().min(-90).max(90), - longitude: z.number().min(-180).max(180), - accuracy: z.number().positive().optional(), - }) - .optional(), - browserId: BrowserAutomationBrowserIdSchema, - }, - outputSchema: BrowserToolOutputSchema, - }, - async ({ viewport, geolocation, browserId }) => { - const context = resolveBrowserToolContext(options); - const payload = await options.broker.execute({ - agentId: context.agentId, - cwd: context.cwd, - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - - command: { - command: "environment", - args: { - browserId, - ...(viewport ? { viewport } : {}), - ...(geolocation ? { geolocation } : {}), - }, - }, - }); - return browserToolResult({ payload, context: { ...context, browserId } }); - }, - ); - - options.registerTool( - "browser_set_background", - { - title: "Set browser background", - description: - "Set the current page background color in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.", - inputSchema: { - color: z.string().min(1), - browserId: BrowserAutomationBrowserIdSchema, - }, - outputSchema: BrowserToolOutputSchema, - }, - async ({ color, browserId }) => { - const context = resolveBrowserToolContext(options); - const payload = await options.broker.execute({ - agentId: context.agentId, - cwd: context.cwd, - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - - command: { - command: "set_background", - args: { - browserId, - color, - }, - }, - }); - return browserToolResult({ payload, context: { ...context, browserId } }); - }, - ); } function resolveBrowserToolContext(options: RegisterBrowserToolsOptions): { @@ -975,7 +707,7 @@ function browserToolSuccessContent( function browserToolImageContent( result: Extract["result"], ): PaseoToolResult["content"][number] | null { - if (result.command !== "screenshot" && result.command !== "full_page_screenshot") { + if (result.command !== "screenshot") { return null; } @@ -1004,16 +736,6 @@ function summarizeBrowserSuccess( return diagnosticsSummary; } - const storageSummary = summarizeBrowserStorageSuccess(payload.result); - if (storageSummary) { - return storageSummary; - } - - const environmentSummary = summarizeBrowserEnvironmentSuccess(payload.result); - if (environmentSummary) { - return environmentSummary; - } - const keyboardSummary = summarizeBrowserKeyboardSuccess(payload.result); if (keyboardSummary) { return keyboardSummary; @@ -1057,10 +779,6 @@ function summarizeBrowserSuccess( return `Browser wait matched ${payload.result.matched}.`; } - if (payload.result.command === "page_info") { - return `Current page browserId=${payload.result.tab.browserId}: ${payload.result.tab.title || "Untitled"} — ${payload.result.tab.url}`; - } - return `Browser ${payload.result.command} complete.`; } @@ -1070,15 +788,6 @@ function summarizeBrowserMediaSuccess( if (result.command === "screenshot") { return `Captured browser screenshot (${result.width}x${result.height}).`; } - if (result.command === "full_page_screenshot") { - return `Captured full-page browser screenshot (${result.width}x${result.height}).`; - } - if (result.command === "pdf") { - return "Exported browser page PDF."; - } - if (result.command === "download") { - return `Downloaded browser file to ${result.filePath}.`; - } if (result.command === "upload") { const count = result.filePaths.length; return `Uploaded ${count} file${count === 1 ? "" : "s"} to browser element ${result.ref}.`; @@ -1129,24 +838,6 @@ function summarizeBrowserDiagnosticsSuccess( return `Read ${consoleCount} console log${consoleCount === 1 ? "" : "s"} and ${networkCount} network entr${networkCount === 1 ? "y" : "ies"}.`; } -function summarizeBrowserStorageSuccess( - result: Extract["result"], -): string | null { - if (result.command !== "storage") { - return null; - } - return `Read ${result.cookies.length} cookie${result.cookies.length === 1 ? "" : "s"}, ${result.localStorage.length} localStorage entr${result.localStorage.length === 1 ? "y" : "ies"}, and ${result.sessionStorage.length} sessionStorage entr${result.sessionStorage.length === 1 ? "y" : "ies"}.`; -} - -function summarizeBrowserEnvironmentSuccess( - result: Extract["result"], -): string | null { - if (result.command !== "environment") { - return null; - } - return `Browser environment viewport is ${result.viewport.width}x${result.viewport.height}.`; -} - function summarizeBrowserRefActionSuccess( result: Extract["result"], ): string | null { @@ -1164,18 +855,6 @@ function summarizeBrowserRefActionSuccess( function summarizeBrowserControlSuccess( result: Extract["result"], ): string | null { - if (result.command === "focus") { - return `Focused browser element ${result.ref}.`; - } - - if (result.command === "clear") { - return `Cleared browser element ${result.ref}.`; - } - - if (result.command === "check") { - return `${result.checked ? "Checked" : "Unchecked"} browser element ${result.ref}.`; - } - if (result.command === "select") { return `Selected ${result.value} in browser element ${result.ref}.`; } @@ -1188,10 +867,6 @@ function summarizeBrowserControlSuccess( return `Dragged browser element ${result.sourceRef} to ${result.targetRef}.`; } - if (result.command === "set_background") { - return `Set browser page background to ${result.color}.`; - } - return null; }