jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(browser-tools): cut low-value tools

Browse Source
This commit is contained in:
Mohamed Boudra
2026-07-02 23:56:20 +02:00
parent b2714ccd89
commit 2ba3ad53ab
13 changed files with 119 additions and 1796 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/desktop/src/features/browser-automation/ipc.test.ts
-18
View File
@@ -1,18 +0,0 @@
import { describe, expect, it } from "vitest";
import { sanitizeDownloadFileName } from "./ipc.js";
describe("browser automation IPC", () => {
it("strips directories from agent-supplied download filenames", () => {
expect(
sanitizeDownloadFileName({
url: "https://example.com/fallback.txt",
fileName: "../../.ssh/authorized_keys",
}),
).toBe("authorized_keys");
});
it("falls back to a safe filename when the URL has no basename", () => {
expect(sanitizeDownloadFileName({ url: "https://example.com/" })).toBe("download");
});
});
packages/desktop/src/features/browser-automation/ipc.ts
+1 -56
View File
@@ -1,13 +1,7 @@
import { mkdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import type { WebContents } from "electron";
import { ipcMain } from "electron";
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type {
BrowserAutomationConsoleLogEntry,
BrowserAutomationCookieEntry,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { TabContents, BrowserRegistry } from "./service.js";
import { executeAutomationCommand } from "./service.js";
import {
@@ -46,61 +40,12 @@ function adaptWebContents(contents: WebContents): TabContents {
isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(),
setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed),
getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
getCookies: async (url: string) =>
(await contents.session.cookies.get({ url })).map(normalizeCookie),
sendDebugCommand: async (command: string, params?: Record<string, unknown>) => {
if (!contents.debugger.isAttached()) {
contents.debugger.attach("1.3");
}
return contents.debugger.sendCommand(command, params ?? {});
},
printToPDF: async (options?: Record<string, unknown>) => contents.printToPDF(options ?? {}),
downloadURL: (input) => downloadWithContents(contents, input),
};
}
function downloadWithContents(
contents: WebContents,
input: { url: string; fileName?: string },
): Promise<{ filePath: string; totalBytes?: number; state: string }> {
const downloadDir = join(tmpdir(), "paseo-browser-downloads");
mkdirSync(downloadDir, { recursive: true });
const filePath = join(downloadDir, sanitizeDownloadFileName(input));
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
contents.session.off("will-download", onDownload);
reject(new Error(`Timed out waiting for browser download: ${input.url}`));
}, 30_000);
function onDownload(_event: Electron.Event, item: Electron.DownloadItem): void {
if (item.getURL() !== input.url) {
return;
}
clearTimeout(timeout);
contents.session.off("will-download", onDownload);
item.setSavePath(filePath);
item.once("done", (_doneEvent, state) => {
resolve({ filePath, totalBytes: item.getTotalBytes(), state });
});
}
contents.session.on("will-download", onDownload);
contents.downloadURL(input.url);
});
}
export function sanitizeDownloadFileName(input: { url: string; fileName?: string }): string {
const requestedName = input.fileName ?? basename(new URL(input.url).pathname);
return basename(requestedName) || "download";
}
function normalizeCookie(cookie: Electron.Cookie): BrowserAutomationCookieEntry {
return {
name: cookie.name,
value: cookie.value,
...(cookie.domain ? { domain: cookie.domain } : {}),
...(cookie.path ? { path: cookie.path } : {}),
secure: cookie.secure,
httpOnly: cookie.httpOnly,
...(typeof cookie.expirationDate === "number" ? { expirationDate: cookie.expirationDate } : {}),
};
}
packages/desktop/src/features/browser-automation/service.test.ts
+33 -230
View File
@@ -4,7 +4,6 @@ import { describe, expect, test, vi } from "vitest";
import type {
BrowserAutomationCommand,
BrowserAutomationConsoleLogEntry,
BrowserAutomationCookieEntry,
BrowserAutomationExecuteRequest,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { BrowserSnapshotEngine } from "./snapshot-engine.js";
@@ -37,25 +36,14 @@ class FakeTab implements TabContents {
public readonly actions: string[] = [];
public readonly capturedViewports: Array<{ stayHidden?: boolean }> = [];
public readonly debugCommands: Array<{ command: string; params?: Record<string, unknown> }> = [];
public readonly pdfOptions: Record<string, unknown>[] = [];
public readonly downloads: Array<{ url: string; fileName?: string }> = [];
public destroyed = false;
public bodyText = "";
public snapshotElements: unknown[] = [];
public actionScriptResult: unknown = true;
public networkEntries: unknown[] = [];
public storageState: unknown = { localStorage: [], sessionStorage: [] };
public viewport = { width: 1024, height: 768, deviceScaleFactor: 2 };
public consoleMessages: BrowserAutomationConsoleLogEntry[] = [];
public cookies: BrowserAutomationCookieEntry[] = [];
public captureNeverPaints = false;
public pdfBytes = new Uint8Array([37, 80, 68, 70]);
public downloadResult = {
filePath: "/workspace/downloads/file.txt",
totalBytes: 42,
state: "completed",
};
public layoutMetrics = {
cssLayoutViewport: { clientWidth: 390, clientHeight: 844 },
cssContentSize: { width: 390, height: 1200 },
@@ -106,12 +94,6 @@ class FakeTab implements TabContents {
if (code.includes("performance.getEntriesByType")) {
return JSON.stringify(this.networkEntries);
}
if (code.includes("localStorage") && code.includes("sessionStorage")) {
return JSON.stringify(this.storageState);
}
if (code.includes("window.innerWidth")) {
return JSON.stringify(this.viewport);
}
return this.actionScriptResult;
}
@@ -157,10 +139,6 @@ class FakeTab implements TabContents {
return this.consoleMessages;
}
public async getCookies(_url: string): Promise<BrowserAutomationCookieEntry[]> {
return this.cookies;
}
public async sendDebugCommand(
command: string,
params?: Record<string, unknown>,
@@ -180,20 +158,6 @@ class FakeTab implements TabContents {
}
return {};
}
public async printToPDF(options?: Record<string, unknown>): Promise<Uint8Array> {
this.pdfOptions.push(options ?? {});
return this.pdfBytes;
}
public async downloadURL(input: { url: string; fileName?: string }): Promise<{
filePath: string;
totalBytes?: number;
state: string;
}> {
this.downloads.push(input);
return this.downloadResult;
}
}
class FakeRegistry implements BrowserRegistry {
@@ -470,50 +434,20 @@ describe("executeAutomationCommand", () => {
});
});
test("page info reads the explicit browser id from command args", () => {
const browser = new BrowserAutomationHarness();
const result = executeAutomationCommand(
automationRequest(
{ command: "page_info", args: { browserId: BROWSER_A } },
{ requestId: "req-page" },
),
browser.registry,
);
expect(result).toEqual({
requestId: "req-page",
ok: true,
result: {
command: "page_info",
tab: {
browserId: BROWSER_A,
workspaceId: WORKSPACE_A,
url: "https://a.test/form",
title: "Fixture",
isActive: true,
isLoading: false,
canGoBack: true,
canGoForward: false,
},
},
});
});
test("page info returns tab not found for an id in another workspace", () => {
test("tab commands return tab not found for an id in another workspace", async () => {
const registry = new FakeRegistry();
registry.register(BROWSER_A, WORKSPACE_B, new FakeTab(1, "https://a.test", "A"));
const result = executeAutomationCommand(
const result = await executeAutomationCommand(
automationRequest(
{ command: "page_info", args: { browserId: BROWSER_A } },
{ requestId: "req-page" },
{ command: "snapshot", args: { browserId: BROWSER_A } },
{ requestId: "req-snapshot" },
),
registry,
);
expect(result).toEqual({
requestId: "req-page",
requestId: "req-snapshot",
ok: false,
error: {
code: "browser_tab_not_found",
@@ -523,22 +457,22 @@ describe("executeAutomationCommand", () => {
});
});
test("page info returns tab closed for a destroyed explicit tab", () => {
test("tab commands return tab closed for a destroyed explicit tab", async () => {
const tab = new FakeTab(1, "https://a.test", "A");
tab.destroyed = true;
const registry = new FakeRegistry();
registry.register(BROWSER_A, WORKSPACE_A, tab);
const result = executeAutomationCommand(
const result = await executeAutomationCommand(
automationRequest(
{ command: "page_info", args: { browserId: BROWSER_A } },
{ requestId: "req-page" },
{ command: "snapshot", args: { browserId: BROWSER_A } },
{ requestId: "req-snapshot" },
),
registry,
);
expect(result).toEqual({
requestId: "req-page",
requestId: "req-snapshot",
ok: false,
error: {
code: "browser_tab_closed",
@@ -595,22 +529,6 @@ describe("executeAutomationCommand", () => {
expect(containsScript(browser.tab, "#submit", ".click()")).toBe(true);
});
test("set background writes the requested color into the explicit page", async () => {
const browser = new BrowserAutomationHarness();
const result = await browser.execute({
command: "set_background",
args: { browserId: BROWSER_A, color: "red" },
});
expect(result).toEqual({
requestId: "req-set_background",
ok: true,
result: { command: "set_background", browserId: BROWSER_A, color: "red" },
});
expect(containsScript(browser.tab, "document.body.style.background", "red")).toBe(true);
});
test.each([
{
name: "fill updates a ref from the latest snapshot",
@@ -618,24 +536,6 @@ describe("executeAutomationCommand", () => {
result: { command: "fill", browserId: BROWSER_A, ref: "@e1" },
scriptParts: ["#name", "Ada"],
},
{
name: "focus focuses a ref from the latest snapshot",
command: { command: "focus", args: { browserId: BROWSER_A, ref: "@e1" } },
result: { command: "focus", browserId: BROWSER_A, ref: "@e1" },
scriptParts: ["#name", "focus"],
},
{
name: "clear clears a ref from the latest snapshot",
command: { command: "clear", args: { browserId: BROWSER_A, ref: "@e1" } },
result: { command: "clear", browserId: BROWSER_A, ref: "@e1" },
scriptParts: ["#name", "deleteContent"],
},
{
name: "check sets the requested checked state on a ref",
command: { command: "check", args: { browserId: BROWSER_A, ref: "@e2", checked: false } },
result: { command: "check", browserId: BROWSER_A, ref: "@e2", checked: false },
scriptParts: ["#agree", "nextChecked = false"],
},
{
name: "select sets the requested value on a ref",
command: { command: "select", args: { browserId: BROWSER_A, ref: "@e3", value: "us" } },
@@ -672,6 +572,24 @@ describe("executeAutomationCommand", () => {
expect(containsScript(browser.tab, ...scriptParts)).toBe(true);
});
test("fill with an empty string clears a ref through the regular fill path", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.snapshotElements = formElements();
requireSnapshotRefs(await browser.snapshot());
const action = await browser.execute({
command: "fill",
args: { browserId: BROWSER_A, ref: "@e1", value: "" },
});
expect(action).toEqual({
requestId: "req-fill",
ok: true,
result: { command: "fill", browserId: BROWSER_A, ref: "@e1" },
});
expect(containsScript(browser.tab, "#name", 'const nextValue = "";')).toBe(true);
});
test("refs become stale after navigation changes the tab URL", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.snapshotElements = formElements();
@@ -912,71 +830,6 @@ describe("executeAutomationCommand", () => {
});
});
test("storage reads cookies and web storage from the explicit tab", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.cookies = [
{ name: "theme", value: "dark", domain: "a.test", path: "/", secure: true },
];
browser.tab.storageState = {
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
};
const result = await browser.execute({
command: "storage",
args: { browserId: BROWSER_A },
});
expect(result).toEqual({
requestId: "req-storage",
ok: true,
result: {
command: "storage",
browserId: BROWSER_A,
url: "https://a.test/form",
cookies: [{ name: "theme", value: "dark", domain: "a.test", path: "/", secure: true }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
});
});
test("environment applies viewport and geolocation before reporting the current viewport", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.viewport = { width: 390, height: 844, deviceScaleFactor: 3 };
const result = await browser.execute({
command: "environment",
args: {
browserId: BROWSER_A,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
});
expect(result).toEqual({
requestId: "req-environment",
ok: true,
result: {
command: "environment",
browserId: BROWSER_A,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
});
expect(browser.tab.debugCommands).toEqual([
{
command: "Emulation.setDeviceMetricsOverride",
params: { width: 390, height: 844, deviceScaleFactor: 3, mobile: false },
},
{
command: "Emulation.setGeolocationOverride",
params: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
]);
expect(containsScript(browser.tab, "navigator", "geolocation")).toBe(true);
});
test("screenshot serializes the painted viewport and restores throttling", async () => {
const browser = new BrowserAutomationHarness();
@@ -1039,19 +892,19 @@ describe("executeAutomationCommand", () => {
}
});
test("full page screenshot captures the page content area through CDP", async () => {
test("screenshot with fullPage captures the page content area through CDP", async () => {
const browser = new BrowserAutomationHarness();
const result = await browser.execute({
command: "full_page_screenshot",
args: { browserId: BROWSER_A },
command: "screenshot",
args: { browserId: BROWSER_A, fullPage: true },
});
expect(result).toEqual({
requestId: "req-full_page_screenshot",
requestId: "req-screenshot",
ok: true,
result: {
command: "full_page_screenshot",
command: "screenshot",
browserId: BROWSER_A,
mimeType: "image/png",
dataBase64: "fullPagePng",
@@ -1072,56 +925,6 @@ describe("executeAutomationCommand", () => {
]);
});
test("pdf exports the explicit tab with requested print options", async () => {
const browser = new BrowserAutomationHarness();
const result = await browser.execute({
command: "pdf",
args: { browserId: BROWSER_A, landscape: true, printBackground: false },
});
expect(result).toEqual({
requestId: "req-pdf",
ok: true,
result: {
command: "pdf",
browserId: BROWSER_A,
mimeType: "application/pdf",
dataBase64: "JVBERg==",
},
});
expect(browser.tab.pdfOptions).toEqual([{ printBackground: false, landscape: true }]);
});
test("download saves the requested HTTP URL through the explicit tab", async () => {
const browser = new BrowserAutomationHarness();
const result = await browser.execute({
command: "download",
args: {
browserId: BROWSER_A,
url: "https://a.test/file.txt",
fileName: "file.txt",
},
});
expect(result).toEqual({
requestId: "req-download",
ok: true,
result: {
command: "download",
browserId: BROWSER_A,
url: "https://a.test/file.txt",
filePath: "/workspace/downloads/file.txt",
totalBytes: 42,
state: "completed",
},
});
expect(browser.tab.downloads).toEqual([
{ url: "https://a.test/file.txt", fileName: "file.txt" },
]);
});
test("upload resolves workspace files before setting them on the file input", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.snapshotElements = [
packages/desktop/src/features/browser-automation/service.ts
+10 -485
View File
@@ -3,12 +3,10 @@ import { isAbsolute, relative, resolve as resolvePath } from "node:path";
import type {
BrowserAutomationCommand,
BrowserAutomationConsoleLogEntry,
BrowserAutomationCookieEntry,
BrowserAutomationErrorCode,
BrowserAutomationExecuteResponse,
BrowserAutomationExecuteRequest,
BrowserAutomationNetworkLogEntry,
BrowserAutomationStorageEntry,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { BrowserSnapshotEngine } from "./snapshot-engine.js";
@@ -30,14 +28,7 @@ export interface TabContents {
isBackgroundThrottlingAllowed(): boolean;
setBackgroundThrottling(allowed: boolean): void;
getConsoleMessages?(): BrowserAutomationConsoleLogEntry[];
getCookies?(url: string): Promise<BrowserAutomationCookieEntry[]>;
sendDebugCommand?(command: string, params?: Record<string, unknown>): Promise<unknown>;
printToPDF?(options?: Record<string, unknown>): Promise<Uint8Array>;
downloadURL?(input: { url: string; fileName?: string }): Promise<{
filePath: string;
totalBytes?: number;
state: string;
}>;
}
export interface TabImage {
@@ -195,10 +186,6 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
executeListTabs(requestId, workspaceId, registry),
new_tab: ({ requestId }) =>
fail(requestId, "browser_unsupported", "browser_new_tab is handled by the app runtime."),
page_info: ({ command, requestId, workspaceId, registry }) => {
const pageInfoCommand = command as Extract<BrowserAutomationCommand, { command: "page_info" }>;
return executePageInfo(requestId, workspaceId, pageInfoCommand.args.browserId, registry);
},
snapshot: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const snapshotCommand = command as Extract<BrowserAutomationCommand, { command: "snapshot" }>;
return executeSnapshot(
@@ -315,37 +302,11 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
BrowserAutomationCommand,
{ command: "screenshot" }
>;
return executeScreenshot(requestId, workspaceId, screenshotCommand.args.browserId, registry);
},
full_page_screenshot: ({ command, requestId, workspaceId, registry }) => {
const screenshotCommand = command as Extract<
BrowserAutomationCommand,
{ command: "full_page_screenshot" }
>;
return executeFullPageScreenshot(
return executeScreenshot(
requestId,
workspaceId,
screenshotCommand.args.browserId,
registry,
);
},
pdf: ({ command, requestId, workspaceId, registry }) => {
const pdfCommand = command as Extract<BrowserAutomationCommand, { command: "pdf" }>;
return executePdf(
requestId,
workspaceId,
pdfCommand.args.browserId,
{ landscape: pdfCommand.args.landscape, printBackground: pdfCommand.args.printBackground },
registry,
);
},
download: ({ command, requestId, workspaceId, registry }) => {
const downloadCommand = command as Extract<BrowserAutomationCommand, { command: "download" }>;
return executeDownload(
requestId,
workspaceId,
downloadCommand.args.browserId,
{ url: downloadCommand.args.url, fileName: downloadCommand.args.fileName },
screenshotCommand.args.fullPage,
registry,
);
},
@@ -361,40 +322,6 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
snapshotEngine,
);
},
focus: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const focusCommand = command as Extract<BrowserAutomationCommand, { command: "focus" }>;
return executeFocus(
requestId,
workspaceId,
focusCommand.args.browserId,
focusCommand.args.ref,
registry,
snapshotEngine,
);
},
clear: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const clearCommand = command as Extract<BrowserAutomationCommand, { command: "clear" }>;
return executeClear(
requestId,
workspaceId,
clearCommand.args.browserId,
clearCommand.args.ref,
registry,
snapshotEngine,
);
},
check: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const checkCommand = command as Extract<BrowserAutomationCommand, { command: "check" }>;
return executeCheck(
requestId,
workspaceId,
checkCommand.args.browserId,
checkCommand.args.ref,
checkCommand.args.checked,
registry,
snapshotEngine,
);
},
select: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const selectCommand = command as Extract<BrowserAutomationCommand, { command: "select" }>;
return executeSelect(
@@ -440,39 +367,6 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
registry,
);
},
storage: ({ command, requestId, workspaceId, registry }) => {
const storageCommand = command as Extract<BrowserAutomationCommand, { command: "storage" }>;
return executeStorage(requestId, workspaceId, storageCommand.args.browserId, registry);
},
environment: ({ command, requestId, workspaceId, registry }) => {
const environmentCommand = command as Extract<
BrowserAutomationCommand,
{ command: "environment" }
>;
return executeEnvironment(
requestId,
workspaceId,
environmentCommand.args.browserId,
{
viewport: environmentCommand.args.viewport,
geolocation: environmentCommand.args.geolocation,
},
registry,
);
},
set_background: ({ command, requestId, workspaceId, registry }) => {
const setBackgroundCommand = command as Extract<
BrowserAutomationCommand,
{ command: "set_background" }
>;
return executeSetBackground(
requestId,
workspaceId,
setBackgroundCommand.args.browserId,
setBackgroundCommand.args.color,
registry,
);
},
};
interface ResolvedTabTarget {
@@ -508,37 +402,6 @@ function executeListTabs(
return { requestId, ok: true, result: { command: "list_tabs", tabs } };
}
function executePageInfo(
requestId: string,
workspaceId: string | undefined,
browserId: string,
registry: BrowserRegistry,
): AutomationCommandPayload {
const target = resolveTabTarget({
requestId,
workspaceId,
browserId,
registry,
});
if ("ok" in target) {
return target;
}
return {
requestId,
ok: true,
result: {
command: "page_info",
tab: tabInfoFromContents(
target.browserId,
target.contents,
workspaceId ? registry.getWorkspaceActiveBrowserId(workspaceId) : null,
registry.getBrowserWorkspaceId(target.browserId),
),
},
};
}
async function executeSnapshot(
requestId: string,
workspaceId: string | undefined,
@@ -625,81 +488,6 @@ async function executeFill(
return { requestId, ok: true, result: { command: "fill", browserId: target.browserId, ref } };
}
async function executeFocus(
requestId: string,
workspaceId: string | undefined,
browserId: string,
ref: string,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
const result = await snapshotEngine.focus({
browserId: target.browserId,
page: target.contents,
ref,
});
if (!result.ok) {
return staleRefFailure(requestId, ref);
}
return { requestId, ok: true, result: { command: "focus", browserId: target.browserId, ref } };
}
async function executeClear(
requestId: string,
workspaceId: string | undefined,
browserId: string,
ref: string,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
const result = await snapshotEngine.clear({
browserId: target.browserId,
page: target.contents,
ref,
});
if (!result.ok) {
return staleRefFailure(requestId, ref);
}
return { requestId, ok: true, result: { command: "clear", browserId: target.browserId, ref } };
}
async function executeCheck(
requestId: string,
workspaceId: string | undefined,
browserId: string,
ref: string,
checked: boolean,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
const result = await snapshotEngine.check({
browserId: target.browserId,
page: target.contents,
ref,
checked,
});
if (!result.ok) {
return staleRefFailure(requestId, ref);
}
return {
requestId,
ok: true,
result: { command: "check", browserId: target.browserId, ref, checked },
};
}
async function executeSelect(
requestId: string,
workspaceId: string | undefined,
@@ -808,84 +596,6 @@ async function executeLogs(
};
}
async function executeStorage(
requestId: string,
workspaceId: string | undefined,
browserId: string,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
const url = target.contents.getURL();
const cookies = (await target.contents.getCookies?.(url)) ?? [];
const storage = parseStorageState(await target.contents.executeJavaScript(STORAGE_STATE_SCRIPT));
return {
requestId,
ok: true,
result: {
command: "storage",
browserId: target.browserId,
url,
cookies,
localStorage: storage.localStorage,
sessionStorage: storage.sessionStorage,
},
};
}
async function executeEnvironment(
requestId: string,
workspaceId: string | undefined,
browserId: string,
environment: {
viewport?: { width: number; height: number; deviceScaleFactor?: number };
geolocation?: { latitude: number; longitude: number; accuracy?: number };
},
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
if (environment.viewport) {
await target.contents.sendDebugCommand?.("Emulation.setDeviceMetricsOverride", {
width: environment.viewport.width,
height: environment.viewport.height,
deviceScaleFactor: environment.viewport.deviceScaleFactor ?? 1,
mobile: false,
});
}
if (environment.geolocation) {
const geolocation = {
latitude: environment.geolocation.latitude,
longitude: environment.geolocation.longitude,
accuracy: environment.geolocation.accuracy ?? 1,
};
await target.contents.sendDebugCommand?.("Emulation.setGeolocationOverride", geolocation);
await target.contents.executeJavaScript(buildGeolocationShimScript(geolocation));
}
const viewport = parseViewport(await target.contents.executeJavaScript(VIEWPORT_SCRIPT));
return {
requestId,
ok: true,
result: {
command: "environment",
browserId: target.browserId,
viewport,
...(environment.geolocation
? {
geolocation: {
...environment.geolocation,
accuracy: environment.geolocation.accuracy ?? 1,
},
}
: {}),
},
};
}
function staleRefFailure(requestId: string, ref: string): FailurePayload {
return fail(
requestId,
@@ -952,39 +662,6 @@ async function executeWait(
return fail(requestId, "browser_unsupported", "browser_wait requires text or url");
}
async function executeSetBackground(
requestId: string,
workspaceId: string | undefined,
browserId: string,
color: string,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
await target.contents.executeJavaScript(buildSetBackgroundScript(color));
return {
requestId,
ok: true,
result: { command: "set_background", browserId: target.browserId, color },
};
}
function buildSetBackgroundScript(color: string): string {
return String.raw`(() => {
const color = ${JSON.stringify(color)};
document.documentElement.style.background = color;
if (document.body) {
document.body.style.background = color;
document.body.style.backgroundColor = color;
document.body.style.minHeight = '100vh';
}
return true;
})()`;
}
async function executeType(
requestId: string,
workspaceId: string | undefined,
@@ -1092,8 +769,13 @@ async function executeScreenshot(
requestId: string,
workspaceId: string | undefined,
browserId: string,
fullPage: boolean,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
if (fullPage) {
return executeFullPageScreenshot(requestId, workspaceId, browserId, registry);
}
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
@@ -1176,7 +858,7 @@ async function executeFullPageScreenshot(
return target;
}
if (!target.contents.sendDebugCommand) {
return fail(requestId, "browser_unsupported", "browser_full_page_screenshot requires CDP");
return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP");
}
const metrics = await getCdpLayoutMetrics(target.contents);
const width = metrics.contentWidth;
@@ -1197,13 +879,13 @@ async function executeFullPageScreenshot(
throw error;
}
if (!screenshot.data) {
return fail(requestId, "browser_unsupported", "browser_full_page_screenshot returned no data");
return fail(requestId, "browser_unsupported", "browser_screenshot fullPage returned no data");
}
return {
requestId,
ok: true,
result: {
command: "full_page_screenshot",
command: "screenshot",
browserId: target.browserId,
mimeType: "image/png",
dataBase64: screenshot.data,
@@ -1213,68 +895,6 @@ async function executeFullPageScreenshot(
};
}
async function executePdf(
requestId: string,
workspaceId: string | undefined,
browserId: string,
options: { landscape?: boolean; printBackground: boolean },
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
if (!target.contents.printToPDF) {
return fail(requestId, "browser_unsupported", "browser_pdf requires PDF support");
}
const pdf = await target.contents.printToPDF({
printBackground: options.printBackground,
...(options.landscape !== undefined ? { landscape: options.landscape } : {}),
});
return {
requestId,
ok: true,
result: {
command: "pdf",
browserId: target.browserId,
mimeType: "application/pdf",
dataBase64: Buffer.from(pdf).toString("base64"),
},
};
}
async function executeDownload(
requestId: string,
workspaceId: string | undefined,
browserId: string,
input: { url: string; fileName?: string },
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
if (!isAllowedPageUrl(input.url)) {
return fail(requestId, "browser_denied", "Browser download only supports http and https URLs.");
}
if (!target.contents.downloadURL) {
return fail(requestId, "browser_unsupported", "browser_download requires download support");
}
const download = await target.contents.downloadURL(input);
return {
requestId,
ok: true,
result: {
command: "download",
browserId: target.browserId,
url: input.url,
filePath: download.filePath,
...(download.totalBytes !== undefined ? { totalBytes: download.totalBytes } : {}),
state: download.state,
},
};
}
function isAllowedPageUrl(value: string): boolean {
try {
return ALLOWED_PAGE_URL_PROTOCOLS.has(new URL(value).protocol);
@@ -1408,36 +1028,6 @@ function parseNetworkEntries(value: unknown): BrowserAutomationNetworkLogEntry[]
});
}
function parseStorageState(value: unknown): {
localStorage: BrowserAutomationStorageEntry[];
sessionStorage: BrowserAutomationStorageEntry[];
} {
const parsed = typeof value === "string" ? JSON.parse(value) : value;
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return { localStorage: [], sessionStorage: [] };
}
const record = parsed as Record<string, unknown>;
return {
localStorage: parseStorageEntries(record.localStorage),
sessionStorage: parseStorageEntries(record.sessionStorage),
};
}
function parseStorageEntries(value: unknown): BrowserAutomationStorageEntry[] {
if (!Array.isArray(value)) {
return [];
}
return value.flatMap((entry): BrowserAutomationStorageEntry[] => {
if (!entry || typeof entry !== "object") {
return [];
}
const record = entry as Record<string, unknown>;
const key = readString(record.key);
const itemValue = readString(record.value);
return key !== null && itemValue !== null ? [{ key, value: itemValue }] : [];
});
}
function readString(value: unknown): string | null {
return typeof value === "string" ? value : null;
}
@@ -1446,43 +1036,6 @@ function readNumber(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
function parseViewport(value: unknown): {
width: number;
height: number;
deviceScaleFactor: number;
} {
const parsed = typeof value === "string" ? JSON.parse(value) : value;
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return { width: 0, height: 0, deviceScaleFactor: 1 };
}
const record = parsed as Record<string, unknown>;
return {
width: readNumber(record.width) ?? 0,
height: readNumber(record.height) ?? 0,
deviceScaleFactor: readNumber(record.deviceScaleFactor) ?? 1,
};
}
function buildGeolocationShimScript(geolocation: {
latitude: number;
longitude: number;
accuracy: number;
}): string {
return String.raw`(() => {
const coords = ${JSON.stringify({ ...geolocation, altitude: null, altitudeAccuracy: null, heading: null, speed: null })};
const position = { coords, timestamp: Date.now() };
Object.defineProperty(navigator, 'geolocation', {
configurable: true,
value: {
getCurrentPosition(success) { setTimeout(() => success(position), 0); },
watchPosition(success) { setTimeout(() => success(position), 0); return 1; },
clearWatch() {},
},
});
return true;
})()`;
}
const NETWORK_PERFORMANCE_SCRIPT = String.raw`(() => {
const entries = performance.getEntriesByType('resource')
.concat(performance.getEntriesByType('navigation'))
@@ -1498,34 +1051,6 @@ const NETWORK_PERFORMANCE_SCRIPT = String.raw`(() => {
return JSON.stringify(entries);
})()`;
const STORAGE_STATE_SCRIPT = String.raw`(() => {
function entriesFor(storage) {
const entries = [];
for (let index = 0; index < storage.length; index += 1) {
const key = storage.key(index);
if (key !== null) entries.push({ key, value: storage.getItem(key) || '' });
}
return entries;
}
function safeEntries(readStorage) {
try {
return entriesFor(readStorage());
} catch {
return [];
}
}
return JSON.stringify({
localStorage: safeEntries(() => window.localStorage),
sessionStorage: safeEntries(() => window.sessionStorage),
});
})()`;
const VIEWPORT_SCRIPT = String.raw`(() => JSON.stringify({
width: window.innerWidth,
height: window.innerHeight,
deviceScaleFactor: window.devicePixelRatio || 1,
}))()`;
function resolveTabTarget(input: {
requestId: string;
workspaceId: string | undefined;
packages/desktop/src/features/browser-automation/snapshot-engine.test.ts
+3 -1
View File
@@ -44,7 +44,9 @@ describe("BrowserSnapshotEngine", () => {
ok: false,
reason: "stale_ref",
});
await expect(engine.focus({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({
await expect(
engine.select({ browserId: "browser-1", page, ref: "@e1", value: "us" }),
).resolves.toEqual({
ok: false,
reason: "stale_ref",
});
packages/desktop/src/features/browser-automation/snapshot-engine.ts
-76
View File
@@ -108,31 +108,6 @@ export class BrowserSnapshotEngine {
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
async focus(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildFocusScript(selector));
}
async clear(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildClearScript(selector));
}
async check(input: {
browserId: string;
page: SnapshotPage;
ref: string;
checked: boolean;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildCheckScript(selector, input.checked));
}
async select(input: {
browserId: string;
page: SnapshotPage;
@@ -304,57 +279,6 @@ function buildKeypressScript(selector: string | undefined, key: string): string
})()`;
}
function buildFocusScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
return document.activeElement === element;
})()`;
}
function buildClearScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
if ('value' in element) {
element.value = '';
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
element.textContent = '';
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
return true;
})()`;
}
function buildCheckScript(selector: string, checked: boolean): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
const nextChecked = ${JSON.stringify(checked)};
if ('checked' in element) {
element.checked = nextChecked;
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
if (element.getAttribute('role') === 'checkbox' || element.getAttribute('role') === 'radio') {
element.setAttribute('aria-checked', String(nextChecked));
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
return false;
})()`;
}
function buildSelectScript(selector: string, value: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
packages/protocol/src/browser-automation/rpc-schemas.test.ts
+17 -205
View File
@@ -73,28 +73,12 @@ const commandParseCases = [
{
name: "screenshot",
command: { command: "screenshot", args: { browserId: BROWSER_ID } },
expected: { command: "screenshot", args: { browserId: BROWSER_ID } },
expected: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: false } },
},
{
name: "full page screenshot",
command: { command: "full_page_screenshot", args: { browserId: BROWSER_ID } },
expected: { command: "full_page_screenshot", args: { browserId: BROWSER_ID } },
},
{
name: "pdf",
command: { command: "pdf", args: { browserId: BROWSER_ID } },
expected: { command: "pdf", args: { browserId: BROWSER_ID, printBackground: true } },
},
{
name: "download",
command: {
command: "download",
args: { browserId: BROWSER_ID, url: "https://example.com/file.txt" },
},
expected: {
command: "download",
args: { browserId: BROWSER_ID, url: "https://example.com/file.txt" },
},
command: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: true } },
expected: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: true } },
},
{
name: "upload",
@@ -107,21 +91,6 @@ const commandParseCases = [
args: { browserId: BROWSER_ID, ref: "@e1", filePaths: ["/tmp/file.txt"] },
},
},
{
name: "focus",
command: { command: "focus", args: { browserId: BROWSER_ID, ref: "@e1" } },
expected: { command: "focus", args: { browserId: BROWSER_ID, ref: "@e1" } },
},
{
name: "clear",
command: { command: "clear", args: { browserId: BROWSER_ID, ref: "@e1" } },
expected: { command: "clear", args: { browserId: BROWSER_ID, ref: "@e1" } },
},
{
name: "check",
command: { command: "check", args: { browserId: BROWSER_ID, ref: "@e2" } },
expected: { command: "check", args: { browserId: BROWSER_ID, ref: "@e2", checked: true } },
},
{
name: "select",
command: {
@@ -154,61 +123,9 @@ const commandParseCases = [
command: { command: "logs", args: { browserId: BROWSER_ID } },
expected: { command: "logs", args: { browserId: BROWSER_ID, maxEntries: 50 } },
},
{
name: "storage",
command: { command: "storage", args: { browserId: BROWSER_ID } },
expected: { command: "storage", args: { browserId: BROWSER_ID } },
},
{
name: "environment",
command: {
command: "environment",
args: {
browserId: BROWSER_ID,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
expected: {
command: "environment",
args: {
browserId: BROWSER_ID,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
},
{
name: "set background",
command: { command: "set_background", args: { browserId: BROWSER_ID, color: "red" } },
expected: { command: "set_background", args: { browserId: BROWSER_ID, color: "red" } },
},
] as const;
const resultParseCases = [
{
name: "page info",
result: {
command: "page_info",
tab: {
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
},
},
expected: {
command: "page_info",
tab: {
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
isActive: false,
isLoading: false,
},
},
},
{
name: "snapshot",
result: {
@@ -313,7 +230,7 @@ const resultParseCases = [
{
name: "full page screenshot",
result: {
command: "full_page_screenshot",
command: "screenshot",
browserId: BROWSER_ID,
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
@@ -321,7 +238,7 @@ const resultParseCases = [
height: 1200,
},
expected: {
command: "full_page_screenshot",
command: "screenshot",
browserId: BROWSER_ID,
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
@@ -329,40 +246,6 @@ const resultParseCases = [
height: 1200,
},
},
{
name: "pdf",
result: {
command: "pdf",
browserId: BROWSER_ID,
mimeType: "application/pdf",
dataBase64: "JVBERg==",
},
expected: {
command: "pdf",
browserId: BROWSER_ID,
mimeType: "application/pdf",
dataBase64: "JVBERg==",
},
},
{
name: "download",
result: {
command: "download",
browserId: BROWSER_ID,
url: "https://example.com/file.txt",
filePath: "/tmp/file.txt",
totalBytes: 5,
state: "completed",
},
expected: {
command: "download",
browserId: BROWSER_ID,
url: "https://example.com/file.txt",
filePath: "/tmp/file.txt",
totalBytes: 5,
state: "completed",
},
},
{
name: "upload",
result: {
@@ -378,21 +261,6 @@ const resultParseCases = [
filePaths: ["/tmp/file.txt"],
},
},
{
name: "focus",
result: { command: "focus", browserId: BROWSER_ID, ref: "@e1" },
expected: { command: "focus", browserId: BROWSER_ID, ref: "@e1" },
},
{
name: "clear",
result: { command: "clear", browserId: BROWSER_ID, ref: "@e1" },
expected: { command: "clear", browserId: BROWSER_ID, ref: "@e1" },
},
{
name: "check",
result: { command: "check", browserId: BROWSER_ID, ref: "@e2", checked: false },
expected: { command: "check", browserId: BROWSER_ID, ref: "@e2", checked: false },
},
{
name: "select",
result: { command: "select", browserId: BROWSER_ID, ref: "@e3", value: "us" },
@@ -437,45 +305,6 @@ const resultParseCases = [
],
},
},
{
name: "storage",
result: {
command: "storage",
browserId: BROWSER_ID,
url: "https://example.com",
cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
expected: {
command: "storage",
browserId: BROWSER_ID,
url: "https://example.com",
cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
},
{
name: "environment",
result: {
command: "environment",
browserId: BROWSER_ID,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
expected: {
command: "environment",
browserId: BROWSER_ID,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
{
name: "set background",
result: { command: "set_background", browserId: BROWSER_ID, color: "red" },
expected: { command: "set_background", browserId: BROWSER_ID, color: "red" },
},
] as const;
describe("browser automation execute RPC schemas", () => {
@@ -526,9 +355,9 @@ describe("browser automation execute RPC schemas", () => {
test("tab commands reject hallucinated browser ids", () => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId: "req-page-info",
requestId: "req-snapshot",
workspaceId: "workspace-1",
command: { command: "page_info", args: { browserId: "default" } },
command: { command: "snapshot", args: { browserId: "default" } },
});
expect(parsed).toMatchObject({
@@ -540,13 +369,13 @@ describe("browser automation execute RPC schemas", () => {
test("tab commands parse browser ids produced by the fallback generator", () => {
const parsed = BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-page-info",
requestId: "req-snapshot",
workspaceId: "workspace-1",
command: { command: "page_info", args: { browserId: FALLBACK_BROWSER_ID } },
command: { command: "snapshot", args: { browserId: FALLBACK_BROWSER_ID } },
});
expect(parsed.command).toEqual({
command: "page_info",
command: "snapshot",
args: { browserId: FALLBACK_BROWSER_ID },
});
});
@@ -658,22 +487,6 @@ describe("browser automation execute RPC schemas", () => {
});
});
test("download rejects non-http URLs at the protocol boundary", () => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId: "req-download",
command: {
command: "download",
args: { browserId: BROWSER_ID, url: "file:///tmp/secret.txt" },
},
});
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: "URL must use http or https" })] },
});
});
test("new tab responses declare the generated browser id shape", () => {
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
@@ -705,16 +518,15 @@ describe("browser automation execute RPC schemas", () => {
const parsed = BrowserAutomationExecuteResponseSchema.safeParse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-page-info",
requestId: "req-snapshot",
ok: true,
result: {
command: "page_info",
tab: {
browserId: "default",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
},
command: "snapshot",
browserId: "default",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
elements: [],
},
},
});
packages/protocol/src/browser-automation/rpc-schemas.ts
+1 -199
View File
@@ -55,11 +55,6 @@ export const BrowserAutomationNewTabCommandSchema = z.object({
.default({}),
});
export const BrowserAutomationPageInfoCommandSchema = z.object({
command: z.literal("page_info"),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationSnapshotCommandSchema = z.object({
command: z.literal("snapshot"),
args: BrowserAutomationTabTargetSchema,
@@ -131,27 +126,8 @@ export const BrowserAutomationReloadCommandSchema = z.object({
export const BrowserAutomationScreenshotCommandSchema = z.object({
command: z.literal("screenshot"),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({
command: z.literal("full_page_screenshot"),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationPdfCommandSchema = z.object({
command: z.literal("pdf"),
args: BrowserAutomationTabTargetSchema.extend({
landscape: z.boolean().optional(),
printBackground: z.boolean().default(true),
}),
});
export const BrowserAutomationDownloadCommandSchema = z.object({
command: z.literal("download"),
args: BrowserAutomationTabTargetSchema.extend({
url: BrowserAutomationHttpUrlSchema,
fileName: z.string().min(1).optional(),
fullPage: z.boolean().default(false),
}),
});
@@ -163,28 +139,6 @@ export const BrowserAutomationUploadCommandSchema = z.object({
}),
});
export const BrowserAutomationFocusCommandSchema = z.object({
command: z.literal("focus"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationClearCommandSchema = z.object({
command: z.literal("clear"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationCheckCommandSchema = z.object({
command: z.literal("check"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
checked: z.boolean().default(true),
}),
});
export const BrowserAutomationSelectCommandSchema = z.object({
command: z.literal("select"),
args: BrowserAutomationTabTargetSchema.extend({
@@ -215,42 +169,9 @@ export const BrowserAutomationLogsCommandSchema = z.object({
}),
});
export const BrowserAutomationStorageCommandSchema = z.object({
command: z.literal("storage"),
args: BrowserAutomationTabTargetSchema,
});
const BrowserAutomationViewportInputSchema = z.object({
width: z.number().int().positive(),
height: z.number().int().positive(),
deviceScaleFactor: z.number().positive().optional(),
});
const BrowserAutomationGeolocationInputSchema = z.object({
latitude: z.number().min(-90).max(90),
longitude: z.number().min(-180).max(180),
accuracy: z.number().positive().optional(),
});
export const BrowserAutomationEnvironmentCommandSchema = z.object({
command: z.literal("environment"),
args: BrowserAutomationTabTargetSchema.extend({
viewport: BrowserAutomationViewportInputSchema.optional(),
geolocation: BrowserAutomationGeolocationInputSchema.optional(),
}),
});
export const BrowserAutomationSetBackgroundCommandSchema = z.object({
command: z.literal("set_background"),
args: BrowserAutomationTabTargetSchema.extend({
color: z.string().min(1),
}),
});
export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsCommandSchema,
BrowserAutomationNewTabCommandSchema,
BrowserAutomationPageInfoCommandSchema,
BrowserAutomationSnapshotCommandSchema,
BrowserAutomationClickCommandSchema,
BrowserAutomationFillCommandSchema,
@@ -262,20 +183,11 @@ export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
BrowserAutomationForwardCommandSchema,
BrowserAutomationReloadCommandSchema,
BrowserAutomationScreenshotCommandSchema,
BrowserAutomationFullPageScreenshotCommandSchema,
BrowserAutomationPdfCommandSchema,
BrowserAutomationDownloadCommandSchema,
BrowserAutomationUploadCommandSchema,
BrowserAutomationFocusCommandSchema,
BrowserAutomationClearCommandSchema,
BrowserAutomationCheckCommandSchema,
BrowserAutomationSelectCommandSchema,
BrowserAutomationHoverCommandSchema,
BrowserAutomationDragCommandSchema,
BrowserAutomationLogsCommandSchema,
BrowserAutomationStorageCommandSchema,
BrowserAutomationEnvironmentCommandSchema,
BrowserAutomationSetBackgroundCommandSchema,
]);
export const BrowserAutomationTabInfoSchema = z.object({
@@ -301,11 +213,6 @@ export const BrowserAutomationNewTabResultSchema = z.object({
url: z.string().min(1),
});
export const BrowserAutomationPageInfoResultSchema = z.object({
command: z.literal("page_info"),
tab: BrowserAutomationTabInfoSchema,
});
export const BrowserAutomationSnapshotElementSchema = z.object({
ref: z.string().regex(/^@e\d+$/),
role: z.string(),
@@ -385,31 +292,6 @@ export const BrowserAutomationScreenshotResultSchema = z.object({
height: z.number().int().nonnegative(),
});
export const BrowserAutomationFullPageScreenshotResultSchema = z.object({
command: z.literal("full_page_screenshot"),
browserId: BrowserAutomationBrowserIdSchema,
mimeType: z.literal("image/png"),
dataBase64: z.string().min(1),
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
});
export const BrowserAutomationPdfResultSchema = z.object({
command: z.literal("pdf"),
browserId: BrowserAutomationBrowserIdSchema,
mimeType: z.literal("application/pdf"),
dataBase64: z.string().min(1),
});
export const BrowserAutomationDownloadResultSchema = z.object({
command: z.literal("download"),
browserId: BrowserAutomationBrowserIdSchema,
url: z.string().min(1),
filePath: z.string().min(1),
totalBytes: z.number().int().nonnegative().optional(),
state: z.string().min(1),
});
export const BrowserAutomationUploadResultSchema = z.object({
command: z.literal("upload"),
browserId: BrowserAutomationBrowserIdSchema,
@@ -417,25 +299,6 @@ export const BrowserAutomationUploadResultSchema = z.object({
filePaths: z.array(z.string().min(1)).min(1),
});
export const BrowserAutomationFocusResultSchema = z.object({
command: z.literal("focus"),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationClearResultSchema = z.object({
command: z.literal("clear"),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationCheckResultSchema = z.object({
command: z.literal("check"),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
checked: z.boolean(),
});
export const BrowserAutomationSelectResultSchema = z.object({
command: z.literal("select"),
browserId: BrowserAutomationBrowserIdSchema,
@@ -481,59 +344,9 @@ export const BrowserAutomationLogsResultSchema = z.object({
network: z.array(BrowserAutomationNetworkLogEntrySchema),
});
export const BrowserAutomationCookieEntrySchema = z.object({
name: z.string(),
value: z.string(),
domain: z.string().optional(),
path: z.string().optional(),
secure: z.boolean().optional(),
httpOnly: z.boolean().optional(),
expirationDate: z.number().optional(),
});
export const BrowserAutomationStorageEntrySchema = z.object({
key: z.string(),
value: z.string(),
});
export const BrowserAutomationStorageResultSchema = z.object({
command: z.literal("storage"),
browserId: BrowserAutomationBrowserIdSchema,
url: z.string(),
cookies: z.array(BrowserAutomationCookieEntrySchema),
localStorage: z.array(BrowserAutomationStorageEntrySchema),
sessionStorage: z.array(BrowserAutomationStorageEntrySchema),
});
export const BrowserAutomationViewportResultSchema = z.object({
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
deviceScaleFactor: z.number().positive(),
});
export const BrowserAutomationGeolocationResultSchema = z.object({
latitude: z.number(),
longitude: z.number(),
accuracy: z.number(),
});
export const BrowserAutomationEnvironmentResultSchema = z.object({
command: z.literal("environment"),
browserId: BrowserAutomationBrowserIdSchema,
viewport: BrowserAutomationViewportResultSchema,
geolocation: BrowserAutomationGeolocationResultSchema.optional(),
});
export const BrowserAutomationSetBackgroundResultSchema = z.object({
command: z.literal("set_background"),
browserId: BrowserAutomationBrowserIdSchema,
color: z.string().min(1),
});
export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsResultSchema,
BrowserAutomationNewTabResultSchema,
BrowserAutomationPageInfoResultSchema,
BrowserAutomationSnapshotResultSchema,
BrowserAutomationClickResultSchema,
BrowserAutomationFillResultSchema,
@@ -545,20 +358,11 @@ export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
BrowserAutomationForwardResultSchema,
BrowserAutomationReloadResultSchema,
BrowserAutomationScreenshotResultSchema,
BrowserAutomationFullPageScreenshotResultSchema,
BrowserAutomationPdfResultSchema,
BrowserAutomationDownloadResultSchema,
BrowserAutomationUploadResultSchema,
BrowserAutomationFocusResultSchema,
BrowserAutomationClearResultSchema,
BrowserAutomationCheckResultSchema,
BrowserAutomationSelectResultSchema,
BrowserAutomationHoverResultSchema,
BrowserAutomationDragResultSchema,
BrowserAutomationLogsResultSchema,
BrowserAutomationStorageResultSchema,
BrowserAutomationEnvironmentResultSchema,
BrowserAutomationSetBackgroundResultSchema,
]);
export const BrowserAutomationErrorSchema = z.object({
@@ -603,8 +407,6 @@ export type BrowserAutomationConsoleLogEntry = z.infer<
export type BrowserAutomationNetworkLogEntry = z.infer<
typeof BrowserAutomationNetworkLogEntrySchema
>;
export type BrowserAutomationCookieEntry = z.infer<typeof BrowserAutomationCookieEntrySchema>;
export type BrowserAutomationStorageEntry = z.infer<typeof BrowserAutomationStorageEntrySchema>;
export type BrowserAutomationExecuteRequest = z.infer<typeof BrowserAutomationExecuteRequestSchema>;
export type BrowserAutomationExecuteResponse = z.infer<
typeof BrowserAutomationExecuteResponseSchema
packages/protocol/src/messages.browser-automation.test.ts
+1 -1
View File
@@ -41,7 +41,7 @@ describe("browser automation protocol integration", () => {
const parsed = SessionOutboundMessageSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-1",
command: { command: "page_info", args: { browserId } },
command: { command: "snapshot", args: { browserId } },
});
expect(parsed.type).toBe("browser.automation.execute.request");
packages/server/src/server/agent/mcp-server.test.ts
+1 -1
View File
@@ -753,7 +753,7 @@ describe("browser MCP tools", () => {
const response = await tool.handler({});
expect(lookupTool(server, "browser_page_info")).not.toBeUndefined();
expect(lookupTool(server, "browser_snapshot")).not.toBeUndefined();
expect(execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: REPO_CWD,
packages/server/src/server/browser-tools/broker.test.ts
+4 -4
View File
@@ -45,8 +45,8 @@ function createBroker(options: { enabled: boolean; timeoutMs?: number }): Browse
});
}
function pageInfoCommand(): BrowserAutomationCommand {
return { command: "page_info", args: { browserId: BROWSER_ID } };
function snapshotCommand(): BrowserAutomationCommand {
return { command: "snapshot", args: { browserId: BROWSER_ID } };
}
describe("BrowserToolsBroker", () => {
@@ -57,7 +57,7 @@ describe("BrowserToolsBroker", () => {
test("disabled returns browser_disabled", async () => {
const broker = createBroker({ enabled: false });
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
@@ -71,7 +71,7 @@ describe("BrowserToolsBroker", () => {
test("no capable desktop returns browser_no_desktop", async () => {
const broker = createBroker({ enabled: true });
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
packages/server/src/server/browser-tools/tools.test.ts
+40 -187
View File
@@ -68,6 +68,10 @@ class BrowserToolHarness {
return this.get(name).handler(parsed, {});
}
public toolNames(): string[] {
return Array.from(this.tools.keys());
}
private get(name: string): RegisteredTool {
const tool = this.tools.get(name);
if (!tool) {
@@ -150,18 +154,6 @@ function screenshotPayload(): Extract<BrowserToolsResponsePayload, { ok: true }>
}
const routedToolCases = [
{
name: "set background",
toolName: "browser_set_background",
input: { browserId: BROWSER_ID, color: "red" },
command: { command: "set_background", args: { browserId: BROWSER_ID, color: "red" } },
payload: {
requestId: "req-set-background",
ok: true,
result: { command: "set_background", browserId: BROWSER_ID, color: "red" },
},
content: [{ type: "text", text: "Set browser page background to red." }],
},
{
name: "click",
toolName: "browser_click",
@@ -226,7 +218,7 @@ const routedToolCases = [
name: "screenshot",
toolName: "browser_screenshot",
input: { browserId: BROWSER_ID },
command: { command: "screenshot", args: { browserId: BROWSER_ID } },
command: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: false } },
payload: screenshotPayload(),
content: [
{ type: "text", text: "Captured browser screenshot (800x600)." },
@@ -257,65 +249,16 @@ const routedToolCases = [
},
content: [{ type: "text", text: "Read 1 console log and 1 network entry." }],
},
{
name: "storage",
toolName: "browser_storage",
input: { browserId: BROWSER_ID },
command: { command: "storage", args: { browserId: BROWSER_ID } },
payload: {
requestId: "req-storage",
ok: true,
result: {
command: "storage",
browserId: BROWSER_ID,
url: "https://example.com",
cookies: [{ name: "theme", value: "dark" }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
},
content: [
{ type: "text", text: "Read 1 cookie, 1 localStorage entry, and 1 sessionStorage entry." },
],
},
{
name: "environment",
toolName: "browser_environment",
input: {
browserId: BROWSER_ID,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
command: {
command: "environment",
args: {
browserId: BROWSER_ID,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
payload: {
requestId: "req-environment",
ok: true,
result: {
command: "environment",
browserId: BROWSER_ID,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
content: [{ type: "text", text: "Browser environment viewport is 390x844." }],
},
{
name: "full page screenshot",
toolName: "browser_full_page_screenshot",
input: { browserId: BROWSER_ID },
command: { command: "full_page_screenshot", args: { browserId: BROWSER_ID } },
toolName: "browser_screenshot",
input: { browserId: BROWSER_ID, fullPage: true },
command: { command: "screenshot", args: { browserId: BROWSER_ID, fullPage: true } },
payload: {
requestId: "req-full-page",
ok: true,
result: {
command: "full_page_screenshot",
command: "screenshot",
browserId: BROWSER_ID,
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
@@ -324,56 +267,10 @@ const routedToolCases = [
},
},
content: [
{ type: "text", text: "Captured full-page browser screenshot (390x1200)." },
{ type: "text", text: "Captured browser screenshot (390x1200)." },
{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
],
},
{
name: "pdf",
toolName: "browser_pdf",
input: { browserId: BROWSER_ID, landscape: true },
command: {
command: "pdf",
args: { browserId: BROWSER_ID, landscape: true, printBackground: true },
},
payload: {
requestId: "req-pdf",
ok: true,
result: {
command: "pdf",
browserId: BROWSER_ID,
mimeType: "application/pdf",
dataBase64: "JVBERg==",
},
},
content: [{ type: "text", text: "Exported browser page PDF." }],
},
{
name: "download",
toolName: "browser_download",
input: { browserId: BROWSER_ID, url: "https://example.com/file.txt", fileName: "file.txt" },
command: {
command: "download",
args: {
browserId: BROWSER_ID,
url: "https://example.com/file.txt",
fileName: "file.txt",
},
},
payload: {
requestId: "req-download",
ok: true,
result: {
command: "download",
browserId: BROWSER_ID,
url: "https://example.com/file.txt",
filePath: "/tmp/file.txt",
totalBytes: 5,
state: "completed",
},
},
content: [{ type: "text", text: "Downloaded browser file to /tmp/file.txt." }],
},
{
name: "upload",
toolName: "browser_upload",
@@ -394,42 +291,6 @@ const routedToolCases = [
},
content: [{ type: "text", text: "Uploaded 1 file to browser element @e1." }],
},
{
name: "focus",
toolName: "browser_focus",
input: { browserId: BROWSER_ID, ref: "@e1" },
command: { command: "focus", args: { browserId: BROWSER_ID, ref: "@e1" } },
payload: {
requestId: "req-focus",
ok: true,
result: { command: "focus", browserId: BROWSER_ID, ref: "@e1" },
},
content: [{ type: "text", text: "Focused browser element @e1." }],
},
{
name: "clear",
toolName: "browser_clear",
input: { browserId: BROWSER_ID, ref: "@e1" },
command: { command: "clear", args: { browserId: BROWSER_ID, ref: "@e1" } },
payload: {
requestId: "req-clear",
ok: true,
result: { command: "clear", browserId: BROWSER_ID, ref: "@e1" },
},
content: [{ type: "text", text: "Cleared browser element @e1." }],
},
{
name: "check",
toolName: "browser_check",
input: { browserId: BROWSER_ID, ref: "@e2", checked: false },
command: { command: "check", args: { browserId: BROWSER_ID, ref: "@e2", checked: false } },
payload: {
requestId: "req-check",
ok: true,
result: { command: "check", browserId: BROWSER_ID, ref: "@e2", checked: false },
},
content: [{ type: "text", text: "Unchecked browser element @e2." }],
},
{
name: "select",
toolName: "browser_select",
@@ -502,7 +363,7 @@ const brokerErrorCases = [
},
{
name: "typed timeout errors",
toolName: "browser_page_info",
toolName: "browser_snapshot",
input: { browserId: BROWSER_ID },
payload: {
requestId: "req-timeout",
@@ -562,6 +423,31 @@ const brokerErrorCases = [
}>;
describe("registerBrowserTools", () => {
test("registers the kept browser automation tools only", () => {
const harness = new BrowserToolHarness();
expect(harness.toolNames()).toEqual([
"browser_list_tabs",
"browser_new_tab",
"browser_snapshot",
"browser_click",
"browser_fill",
"browser_wait",
"browser_type",
"browser_keypress",
"browser_navigate",
"browser_back",
"browser_forward",
"browser_reload",
"browser_screenshot",
"browser_upload",
"browser_hover",
"browser_select",
"browser_drag",
"browser_logs",
]);
});
test("list tabs sends workspace in the request envelope", async () => {
const harness = new BrowserToolHarness();
@@ -612,12 +498,6 @@ describe("registerBrowserTools", () => {
input: { browserId: BROWSER_ID, url: "localhost:3000" },
expected: { browserId: BROWSER_ID, url: "http://localhost:3000" },
},
{
name: "download accepts an IP host without a scheme as http",
toolName: "browser_download",
input: { browserId: BROWSER_ID, url: "127.0.0.1:8081/admin" },
expected: { browserId: BROWSER_ID, url: "http://127.0.0.1:8081/admin" },
},
{
name: "new tab accepts a domain path without a scheme as http",
toolName: "browser_new_tab",
@@ -662,11 +542,6 @@ describe("registerBrowserTools", () => {
toolName: "browser_navigate",
input: { browserId: BROWSER_ID, url: "file:///tmp/index.html" },
},
{
name: "download rejects file URLs",
toolName: "browser_download",
input: { browserId: BROWSER_ID, url: "file:///tmp/archive.zip" },
},
{
name: "new tab rejects file URLs",
toolName: "browser_new_tab",
@@ -905,38 +780,16 @@ describe("registerBrowserTools", () => {
test("tab tools keep empty context when there is no caller agent", async () => {
const harness = new BrowserToolHarness(null, null);
harness.broker.setResponse({
requestId: "req-page-info",
ok: true,
result: {
command: "page_info",
tab: {
browserId: BROWSER_ID,
url: "https://example.com",
title: "Example",
isActive: false,
isLoading: false,
},
},
});
harness.broker.setResponse(snapshotPayload());
const response = await harness.execute("browser_page_info", { browserId: BROWSER_ID });
const response = await harness.execute("browser_snapshot", { browserId: BROWSER_ID });
expect(harness.broker.calls).toEqual([
{ command: { command: "page_info", args: { browserId: BROWSER_ID } } },
{ command: { command: "snapshot", args: { browserId: BROWSER_ID } } },
]);
expect(response.structuredContent).toEqual({
ok: true,
result: {
command: "page_info",
tab: {
browserId: BROWSER_ID,
url: "https://example.com",
title: "Example",
isActive: false,
isLoading: false,
},
},
result: snapshotPayload().result,
context: { browserId: BROWSER_ID },
});
});
packages/server/src/server/browser-tools/tools.ts
+8 -333
View File
@@ -140,35 +140,6 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
},
);
options.registerTool(
"browser_page_info",
{
title: "Get browser page info",
description:
"Get page info for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: {
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
async ({ browserId }) => {
const context = resolveBrowserToolContext(options);
const payload = await options.broker.execute({
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "page_info",
args: {
browserId,
},
},
});
return browserToolResult({ payload, context: { ...context, browserId } });
},
);
options.registerTool(
"browser_snapshot",
{
@@ -442,11 +413,14 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{
title: "Capture browser screenshot",
description:
"Capture a PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
"Capture a PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs. Set fullPage to true to capture the full page.",
inputSchema: {
browserId: BrowserAutomationBrowserIdSchema,
fullPage: z.boolean().default(false),
},
outputSchema: BrowserToolOutputSchema,
},
async ({ browserId }) => {
async ({ browserId, fullPage }) => {
const context = resolveBrowserToolContext(options);
const payload = await options.broker.execute({
agentId: context.agentId,
@@ -457,99 +431,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
command: "screenshot",
args: {
browserId,
},
},
});
return browserToolResult({ payload, context: { ...context, browserId } });
},
);
options.registerTool(
"browser_full_page_screenshot",
{
title: "Capture full-page browser screenshot",
description:
"Capture a full-page PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema,
},
async ({ browserId }) => {
const context = resolveBrowserToolContext(options);
const payload = await options.broker.execute({
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "full_page_screenshot",
args: {
browserId,
},
},
});
return browserToolResult({ payload, context: { ...context, browserId } });
},
);
options.registerTool(
"browser_pdf",
{
title: "Export browser page PDF",
description:
"Export a Paseo desktop browser tab as a PDF. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: {
browserId: BrowserAutomationBrowserIdSchema,
landscape: z.boolean().optional(),
printBackground: z.boolean().default(true),
},
outputSchema: BrowserToolOutputSchema,
},
async ({ browserId, landscape, printBackground }) => {
const context = resolveBrowserToolContext(options);
const payload = await options.broker.execute({
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "pdf",
args: {
browserId,
...(landscape !== undefined ? { landscape } : {}),
printBackground: printBackground ?? true,
},
},
});
return browserToolResult({ payload, context: { ...context, browserId } });
},
);
options.registerTool(
"browser_download",
{
title: "Download file in browser",
description:
"Download a URL through a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; pass an http(s) URL or a scheme-less host URL, which is treated as http.",
inputSchema: {
url: BrowserHttpUrlInputSchema,
fileName: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
async ({ url, fileName, browserId }) => {
const context = resolveBrowserToolContext(options);
const payload = await options.broker.execute({
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "download",
args: {
browserId,
url,
...(fileName ? { fileName } : {}),
fullPage: fullPage ?? false,
},
},
});
@@ -591,20 +473,6 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
);
for (const toolConfig of [
{
name: "browser_focus",
command: "focus",
title: "Focus browser element",
description:
"Focus an element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
},
{
name: "browser_clear",
command: "clear",
title: "Clear browser element",
description:
"Clear an input-like element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
},
{
name: "browser_hover",
command: "hover",
@@ -641,39 +509,6 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
);
}
options.registerTool(
"browser_check",
{
title: "Check browser control",
description:
"Set a checkbox or radio control in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: {
ref: BrowserRefInputSchema,
checked: z.boolean().default(true),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
async ({ ref, checked, browserId }) => {
const context = resolveBrowserToolContext(options);
const payload = await options.broker.execute({
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "check",
args: {
browserId,
ref,
checked,
},
},
});
return browserToolResult({ payload, context: { ...context, browserId } });
},
);
options.registerTool(
"browser_select",
{
@@ -770,109 +605,6 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
return browserToolResult({ payload, context: { ...context, browserId } });
},
);
options.registerTool(
"browser_storage",
{
title: "Read browser storage",
description:
"Read cookies plus localStorage and sessionStorage for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema,
},
async ({ browserId }) => {
const context = resolveBrowserToolContext(options);
const payload = await options.broker.execute({
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "storage",
args: {
browserId,
},
},
});
return browserToolResult({ payload, context: { ...context, browserId } });
},
);
options.registerTool(
"browser_environment",
{
title: "Set/read browser environment",
description:
"Set or read viewport and geolocation for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: {
viewport: z
.object({
width: z.number().int().positive(),
height: z.number().int().positive(),
deviceScaleFactor: z.number().positive().optional(),
})
.optional(),
geolocation: z
.object({
latitude: z.number().min(-90).max(90),
longitude: z.number().min(-180).max(180),
accuracy: z.number().positive().optional(),
})
.optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
async ({ viewport, geolocation, browserId }) => {
const context = resolveBrowserToolContext(options);
const payload = await options.broker.execute({
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "environment",
args: {
browserId,
...(viewport ? { viewport } : {}),
...(geolocation ? { geolocation } : {}),
},
},
});
return browserToolResult({ payload, context: { ...context, browserId } });
},
);
options.registerTool(
"browser_set_background",
{
title: "Set browser background",
description:
"Set the current page background color in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: {
color: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
async ({ color, browserId }) => {
const context = resolveBrowserToolContext(options);
const payload = await options.broker.execute({
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "set_background",
args: {
browserId,
color,
},
},
});
return browserToolResult({ payload, context: { ...context, browserId } });
},
);
}
function resolveBrowserToolContext(options: RegisterBrowserToolsOptions): {
@@ -975,7 +707,7 @@ function browserToolSuccessContent(
function browserToolImageContent(
result: Extract<BrowserToolsResponsePayload, { ok: true }>["result"],
): PaseoToolResult["content"][number] | null {
if (result.command !== "screenshot" && result.command !== "full_page_screenshot") {
if (result.command !== "screenshot") {
return null;
}
@@ -1004,16 +736,6 @@ function summarizeBrowserSuccess(
return diagnosticsSummary;
}
const storageSummary = summarizeBrowserStorageSuccess(payload.result);
if (storageSummary) {
return storageSummary;
}
const environmentSummary = summarizeBrowserEnvironmentSuccess(payload.result);
if (environmentSummary) {
return environmentSummary;
}
const keyboardSummary = summarizeBrowserKeyboardSuccess(payload.result);
if (keyboardSummary) {
return keyboardSummary;
@@ -1057,10 +779,6 @@ function summarizeBrowserSuccess(
return `Browser wait matched ${payload.result.matched}.`;
}
if (payload.result.command === "page_info") {
return `Current page browserId=${payload.result.tab.browserId}: ${payload.result.tab.title || "Untitled"} — ${payload.result.tab.url}`;
}
return `Browser ${payload.result.command} complete.`;
}
@@ -1070,15 +788,6 @@ function summarizeBrowserMediaSuccess(
if (result.command === "screenshot") {
return `Captured browser screenshot (${result.width}x${result.height}).`;
}
if (result.command === "full_page_screenshot") {
return `Captured full-page browser screenshot (${result.width}x${result.height}).`;
}
if (result.command === "pdf") {
return "Exported browser page PDF.";
}
if (result.command === "download") {
return `Downloaded browser file to ${result.filePath}.`;
}
if (result.command === "upload") {
const count = result.filePaths.length;
return `Uploaded ${count} file${count === 1 ? "" : "s"} to browser element ${result.ref}.`;
@@ -1129,24 +838,6 @@ function summarizeBrowserDiagnosticsSuccess(
return `Read ${consoleCount} console log${consoleCount === 1 ? "" : "s"} and ${networkCount} network entr${networkCount === 1 ? "y" : "ies"}.`;
}
function summarizeBrowserStorageSuccess(
result: Extract<BrowserToolsResponsePayload, { ok: true }>["result"],
): string | null {
if (result.command !== "storage") {
return null;
}
return `Read ${result.cookies.length} cookie${result.cookies.length === 1 ? "" : "s"}, ${result.localStorage.length} localStorage entr${result.localStorage.length === 1 ? "y" : "ies"}, and ${result.sessionStorage.length} sessionStorage entr${result.sessionStorage.length === 1 ? "y" : "ies"}.`;
}
function summarizeBrowserEnvironmentSuccess(
result: Extract<BrowserToolsResponsePayload, { ok: true }>["result"],
): string | null {
if (result.command !== "environment") {
return null;
}
return `Browser environment viewport is ${result.viewport.width}x${result.viewport.height}.`;
}
function summarizeBrowserRefActionSuccess(
result: Extract<BrowserToolsResponsePayload, { ok: true }>["result"],
): string | null {
@@ -1164,18 +855,6 @@ function summarizeBrowserRefActionSuccess(
function summarizeBrowserControlSuccess(
result: Extract<BrowserToolsResponsePayload, { ok: true }>["result"],
): string | null {
if (result.command === "focus") {
return `Focused browser element ${result.ref}.`;
}
if (result.command === "clear") {
return `Cleared browser element ${result.ref}.`;
}
if (result.command === "check") {
return `${result.checked ? "Checked" : "Unchecked"} browser element ${result.ref}.`;
}
if (result.command === "select") {
return `Selected ${result.value} in browser element ${result.ref}.`;
}
@@ -1188,10 +867,6 @@ function summarizeBrowserControlSuccess(
return `Dragged browser element ${result.sourceRef} to ${result.targetRef}.`;
}
if (result.command === "set_background") {
return `Set browser page background to ${result.color}.`;
}
return null;
}