jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(browser-tools): require explicit tab targets

Browse Source 
Remove hidden active-browser targeting so automation requests carry workspace in the envelope and browserId in tab command args.
This commit is contained in:
Mohamed Boudra
2026-07-02 19:53:40 +02:00
parent d9615c1e98
commit 71ce96b60f
22 changed files with 1108 additions and 4227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/architecture.md
+1 -1
View File
@@ -138,7 +138,7 @@ Electron wrapper for macOS, Linux, and Windows.
> **Window-state v1 limitation:** only the _first_ window of a session restores and persists saved geometry (size/position/maximized). Windows opened via ⌘⇧N / second-instance / "Open in new window" open at the default size, OS-cascaded, and do not persist — this avoids every window stacking on the same restored bounds and fighting over the single window-state store. Lifting this needs per-window state keys.
>
> **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus and agent automation targets are intentionally separate: the workspace-active browser follows the user's focused tab, while the agent-active browser is the default target for browser MCP commands. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up.
> **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus still records the workspace-active browser for UI state and `list_tabs` reporting, but agent automation targets only explicit browser ids returned by `browser_new_tab` or `browser_list_tabs`. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up.
### `packages/website` — Marketing site
packages/app/src/browser-automation/handler.test.ts
+1 -7
View File
@@ -73,7 +73,7 @@ function browserNewTabRequest(): BrowserAutomationExecuteRequest {
workspaceId: "wks_workspace_a",
command: {
command: "new_tab",
args: { workspaceId: "wks_workspace_a", url: "https://example.com" },
args: { url: "https://example.com" },
},
};
}
@@ -128,7 +128,6 @@ describe("mountBrowserAutomationHandler", () => {
test("creates a focused workspace browser tab for browser_new_tab", async () => {
const client = new FakeDaemonClient();
const setWorkspaceActiveBrowser = vi.fn(async () => undefined);
const setAgentActiveBrowser = vi.fn(async () => undefined);
const registerWorkspaceBrowser = vi.fn(async () => undefined);
const ensureResidentBrowserWebview = vi.fn();
const executeAutomationCommand = vi.fn(async () => currentListTabsPayload());
@@ -150,7 +149,6 @@ describe("mountBrowserAutomationHandler", () => {
executeAutomationCommand,
registerWorkspaceBrowser,
setWorkspaceActiveBrowser,
setAgentActiveBrowser,
},
}) satisfies DesktopHostBridge,
ensureResidentBrowserWebview,
@@ -180,10 +178,6 @@ describe("mountBrowserAutomationHandler", () => {
browserId,
workspaceId: "wks_workspace_a",
});
expect(setAgentActiveBrowser).toHaveBeenCalledWith({
agentId: "agent-1",
browserId,
});
expect(setWorkspaceActiveBrowser).toHaveBeenCalledWith({
browserId,
workspaceId: "wks_workspace_a",
packages/app/src/browser-automation/handler.ts
+4 -31
View File
@@ -111,8 +111,6 @@ async function handleBrowserAutomationRequest(params: {
return;
}
await rememberAgentBrowserTarget({ request, browserHost });
if (!executeAutomationCommand) {
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
@@ -159,11 +157,11 @@ async function openBrowserTabForRequest(params: {
BrowserAutomationExecuteRequest["command"],
{ command: "new_tab" }
>;
const workspaceId = request.workspaceId ?? command.args.workspaceId;
const workspaceId = request.workspaceId;
if (!serverId || !workspaceId) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_no_tab",
code: "browser_unsupported",
message: "Cannot create a browser tab without a workspace context.",
});
}
@@ -174,7 +172,7 @@ async function openBrowserTabForRequest(params: {
if (!workspaceKey) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_no_tab",
code: "browser_unsupported",
message: "Cannot create a browser tab without a workspace context.",
});
}
@@ -185,9 +183,6 @@ async function openBrowserTabForRequest(params: {
await browserHost?.registerWorkspaceBrowser?.({ browserId, workspaceId });
await browserHost?.setWorkspaceActiveBrowser?.({ browserId, workspaceId });
if (request.agentId) {
await browserHost?.setAgentActiveBrowser?.({ agentId: request.agentId, browserId });
}
if (browserHost?.executeAutomationCommand) {
ensureResidentBrowserWebview({ browserId, url: normalizedUrl });
@@ -236,7 +231,7 @@ async function waitForBrowserRegistration(params: {
agentId: params.request.agentId,
cwd: params.request.cwd,
workspaceId: params.workspaceId,
command: { command: "list_tabs", args: { workspaceId: params.workspaceId } },
command: { command: "list_tabs", args: {} },
});
if (payload.ok && payload.result.command === "list_tabs") {
if (payload.result.tabs.some((tab) => tab.browserId === params.browserId)) {
@@ -252,28 +247,6 @@ function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function rememberAgentBrowserTarget(input: {
request: BrowserAutomationExecuteRequest;
browserHost: DesktopHostBridge["browser"] | undefined;
}): Promise<void> {
if (!input.request.agentId) {
return;
}
const browserId = readRequestBrowserId(input.request);
if (!browserId) {
return;
}
await input.browserHost?.setAgentActiveBrowser?.({ agentId: input.request.agentId, browserId });
}
function readRequestBrowserId(request: BrowserAutomationExecuteRequest): string | null {
if (request.browserId) {
return request.browserId;
}
const args = request.command.args as { browserId?: unknown };
return typeof args.browserId === "string" && args.browserId.length > 0 ? args.browserId : null;
}
function normalizeBridgePayload(
requestId: string,
payload: BrowserAutomationResponsePayload,
packages/app/src/desktop/host.ts
-1
View File
@@ -132,7 +132,6 @@ export interface DesktopBrowserBridge {
workspaceId: string;
browserId: string | null;
}) => Promise<void>;
setAgentActiveBrowser?: (input: { agentId: string; browserId: string | null }) => Promise<void>;
openDevTools?: (browserId: string) => Promise<unknown>;
clearPartition?: (browserId: string) => Promise<void>;
executeAutomationCommand?: (
packages/app/src/stores/browser-store/index.ts
+7 -2
View File
@@ -1,4 +1,5 @@
import AsyncStorage from "@react-native-async-storage/async-storage";
import { BrowserAutomationBrowserIdSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { create } from "zustand";
import { createJSONStorage, persist } from "zustand/middleware";
import {
@@ -22,10 +23,14 @@ interface BrowserStoreState extends BrowserIndexState {
}
function createBrowserId(): string {
let browserId: string;
if (typeof globalThis.crypto?.randomUUID === "function") {
return globalThis.crypto.randomUUID();
browserId = globalThis.crypto.randomUUID();
} else {
const randomSuffix = Math.random().toString(16).slice(2) || "0";
browserId = `${Date.now()}-${randomSuffix}`;
}
return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
return BrowserAutomationBrowserIdSchema.parse(browserId);
}
export const useBrowserStore = create<BrowserStoreState>()(
packages/desktop/src/features/browser-automation/ipc.ts
-7
View File
@@ -14,9 +14,7 @@ import {
listRegisteredPaseoBrowserIds,
listRegisteredPaseoBrowserIdsForWorkspace,
getPaseoBrowserWebContents,
getWorkspaceActivePaseoBrowserWebContents,
getWorkspaceActivePaseoBrowserId,
getAgentActivePaseoBrowserId,
getPaseoBrowserWorkspaceId,
} from "../browser-webviews/index.js";
@@ -149,12 +147,7 @@ function createRegistry(): BrowserRegistry {
return contents ? adaptWebContents(contents) : null;
},
getBrowserWorkspaceId: getPaseoBrowserWorkspaceId,
getWorkspaceActiveTabContents(workspaceId: string): TabContents | null {
const contents = getWorkspaceActivePaseoBrowserWebContents(workspaceId);
return contents ? adaptWebContents(contents) : null;
},
getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId,
getAgentActiveBrowserId: getAgentActivePaseoBrowserId,
};
}
packages/desktop/src/features/browser-automation/service.test.ts
+402 -2173
View File
File diff suppressed because it is too large Load Diff
packages/desktop/src/features/browser-automation/service.ts
+86 -149
View File
@@ -54,9 +54,7 @@ export interface BrowserRegistry {
listRegisteredBrowserIdsForWorkspace(workspaceId: string): string[];
getTabContents(browserId: string): TabContents | null;
getBrowserWorkspaceId(browserId: string): string | null;
getWorkspaceActiveTabContents(workspaceId: string): TabContents | null;
getWorkspaceActiveBrowserId(workspaceId: string): string | null;
getAgentActiveBrowserId(agentId: string): string | null;
}
export type AutomationCommandPayload = BrowserAutomationExecuteResponse["payload"];
@@ -167,45 +165,18 @@ function tabInfoFromContents(
}
export function executeAutomationCommand(
rawRequest: BrowserAutomationExecuteRequest,
request: BrowserAutomationExecuteRequest,
registry: BrowserRegistry,
options?: { snapshotEngine?: BrowserSnapshotEngine },
): AutomationCommandPayload | Promise<AutomationCommandPayload> {
const request = resolveAgentBrowserTarget(rawRequest, registry);
const { requestId, command } = request;
const workspaceId = request.workspaceId ?? command.args.workspaceId;
const workspaceId = request.workspaceId;
const snapshotEngine = options?.snapshotEngine ?? defaultSnapshotEngine;
const handler = commandHandlers[command.command];
return handler({ request, command, requestId, workspaceId, registry, snapshotEngine });
}
function resolveAgentBrowserTarget(
request: BrowserAutomationExecuteRequest,
registry: BrowserRegistry,
): BrowserAutomationExecuteRequest {
if (request.browserId || readCommandBrowserId(request.command)) {
return request;
}
if (!request.agentId) {
return request;
}
const agentBrowserId = registry.getAgentActiveBrowserId(request.agentId);
if (!agentBrowserId) {
return request;
}
return { ...request, browserId: agentBrowserId };
}
function readCommandBrowserId(command: BrowserAutomationCommand): string | undefined {
const args = command.args as { browserId?: unknown };
return typeof args.browserId === "string" && args.browserId.length > 0
? args.browserId
: undefined;
}
interface CommandHandlerContext {
request: BrowserAutomationExecuteRequest;
command: BrowserAutomationCommand;
@@ -224,54 +195,49 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
executeListTabs(requestId, workspaceId, registry),
new_tab: ({ requestId }) =>
fail(requestId, "browser_unsupported", "browser_new_tab is handled by the app runtime."),
page_info: ({ request, command, requestId, workspaceId, registry }) => {
page_info: ({ command, requestId, workspaceId, registry }) => {
const pageInfoCommand = command as Extract<BrowserAutomationCommand, { command: "page_info" }>;
return executePageInfo(
requestId,
workspaceId,
pageInfoCommand.args.browserId ?? request.browserId,
registry,
);
return executePageInfo(requestId, workspaceId, pageInfoCommand.args.browserId, registry);
},
snapshot: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
snapshot: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const snapshotCommand = command as Extract<BrowserAutomationCommand, { command: "snapshot" }>;
return executeSnapshot(
requestId,
workspaceId,
snapshotCommand.args.browserId ?? request.browserId,
snapshotCommand.args.browserId,
registry,
snapshotEngine,
);
},
click: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
click: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const clickCommand = command as Extract<BrowserAutomationCommand, { command: "click" }>;
return executeClick(
requestId,
workspaceId,
clickCommand.args.browserId ?? request.browserId,
clickCommand.args.browserId,
clickCommand.args.ref,
registry,
snapshotEngine,
);
},
fill: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
fill: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const fillCommand = command as Extract<BrowserAutomationCommand, { command: "fill" }>;
return executeFill(
requestId,
workspaceId,
fillCommand.args.browserId ?? request.browserId,
fillCommand.args.browserId,
fillCommand.args.ref,
fillCommand.args.value,
registry,
snapshotEngine,
);
},
wait: ({ request, command, requestId, workspaceId, registry }) => {
wait: ({ command, requestId, workspaceId, registry }) => {
const waitCommand = command as Extract<BrowserAutomationCommand, { command: "wait" }>;
return executeWait(
requestId,
workspaceId,
waitCommand.args.browserId ?? request.browserId,
waitCommand.args.browserId,
{
text: waitCommand.args.text,
url: waitCommand.args.url,
@@ -280,83 +246,78 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
registry,
);
},
type: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
type: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const typeCommand = command as Extract<BrowserAutomationCommand, { command: "type" }>;
return executeType(
requestId,
workspaceId,
typeCommand.args.browserId ?? request.browserId,
typeCommand.args.browserId,
typeCommand.args.ref,
typeCommand.args.text,
registry,
snapshotEngine,
);
},
keypress: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
keypress: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const keypressCommand = command as Extract<BrowserAutomationCommand, { command: "keypress" }>;
return executeKeypress(
requestId,
workspaceId,
keypressCommand.args.browserId ?? request.browserId,
keypressCommand.args.browserId,
keypressCommand.args.ref,
keypressCommand.args.key,
registry,
snapshotEngine,
);
},
navigate: ({ request, command, requestId, workspaceId, registry }) => {
navigate: ({ command, requestId, workspaceId, registry }) => {
const navigateCommand = command as Extract<BrowserAutomationCommand, { command: "navigate" }>;
return executeNavigate(
requestId,
workspaceId,
navigateCommand.args.browserId ?? request.browserId,
navigateCommand.args.browserId,
navigateCommand.args.url,
registry,
);
},
back: ({ request, command, requestId, workspaceId, registry }) => {
back: ({ command, requestId, workspaceId, registry }) => {
const backCommand = command as Extract<BrowserAutomationCommand, { command: "back" }>;
return executeNavigationAction(
requestId,
workspaceId,
backCommand.args.browserId ?? request.browserId,
backCommand.args.browserId,
"back",
registry,
);
},
forward: ({ request, command, requestId, workspaceId, registry }) => {
forward: ({ command, requestId, workspaceId, registry }) => {
const forwardCommand = command as Extract<BrowserAutomationCommand, { command: "forward" }>;
return executeNavigationAction(
requestId,
workspaceId,
forwardCommand.args.browserId ?? request.browserId,
forwardCommand.args.browserId,
"forward",
registry,
);
},
reload: ({ request, command, requestId, workspaceId, registry }) => {
reload: ({ command, requestId, workspaceId, registry }) => {
const reloadCommand = command as Extract<BrowserAutomationCommand, { command: "reload" }>;
return executeNavigationAction(
requestId,
workspaceId,
reloadCommand.args.browserId ?? request.browserId,
reloadCommand.args.browserId,
"reload",
registry,
);
},
screenshot: ({ request, command, requestId, workspaceId, registry }) => {
screenshot: ({ command, requestId, workspaceId, registry }) => {
const screenshotCommand = command as Extract<
BrowserAutomationCommand,
{ command: "screenshot" }
>;
return executeScreenshot(
requestId,
workspaceId,
screenshotCommand.args.browserId ?? request.browserId,
registry,
);
return executeScreenshot(requestId, workspaceId, screenshotCommand.args.browserId, registry);
},
full_page_screenshot: ({ request, command, requestId, workspaceId, registry }) => {
full_page_screenshot: ({ command, requestId, workspaceId, registry }) => {
const screenshotCommand = command as Extract<
BrowserAutomationCommand,
{ command: "full_page_screenshot" }
@@ -364,26 +325,26 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
return executeFullPageScreenshot(
requestId,
workspaceId,
screenshotCommand.args.browserId ?? request.browserId,
screenshotCommand.args.browserId,
registry,
);
},
pdf: ({ request, command, requestId, workspaceId, registry }) => {
pdf: ({ command, requestId, workspaceId, registry }) => {
const pdfCommand = command as Extract<BrowserAutomationCommand, { command: "pdf" }>;
return executePdf(
requestId,
workspaceId,
pdfCommand.args.browserId ?? request.browserId,
pdfCommand.args.browserId,
{ landscape: pdfCommand.args.landscape, printBackground: pdfCommand.args.printBackground },
registry,
);
},
download: ({ request, command, requestId, workspaceId, registry }) => {
download: ({ command, requestId, workspaceId, registry }) => {
const downloadCommand = command as Extract<BrowserAutomationCommand, { command: "download" }>;
return executeDownload(
requestId,
workspaceId,
downloadCommand.args.browserId ?? request.browserId,
downloadCommand.args.browserId,
{ url: downloadCommand.args.url, fileName: downloadCommand.args.fileName },
registry,
);
@@ -394,101 +355,96 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
requestId,
request.cwd,
workspaceId,
uploadCommand.args.browserId ?? request.browserId,
uploadCommand.args.browserId,
{ ref: uploadCommand.args.ref, filePaths: uploadCommand.args.filePaths },
registry,
snapshotEngine,
);
},
focus: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
focus: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const focusCommand = command as Extract<BrowserAutomationCommand, { command: "focus" }>;
return executeFocus(
requestId,
workspaceId,
focusCommand.args.browserId ?? request.browserId,
focusCommand.args.browserId,
focusCommand.args.ref,
registry,
snapshotEngine,
);
},
clear: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
clear: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const clearCommand = command as Extract<BrowserAutomationCommand, { command: "clear" }>;
return executeClear(
requestId,
workspaceId,
clearCommand.args.browserId ?? request.browserId,
clearCommand.args.browserId,
clearCommand.args.ref,
registry,
snapshotEngine,
);
},
check: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
check: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const checkCommand = command as Extract<BrowserAutomationCommand, { command: "check" }>;
return executeCheck(
requestId,
workspaceId,
checkCommand.args.browserId ?? request.browserId,
checkCommand.args.browserId,
checkCommand.args.ref,
checkCommand.args.checked,
registry,
snapshotEngine,
);
},
select: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
select: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const selectCommand = command as Extract<BrowserAutomationCommand, { command: "select" }>;
return executeSelect(
requestId,
workspaceId,
selectCommand.args.browserId ?? request.browserId,
selectCommand.args.browserId,
selectCommand.args.ref,
selectCommand.args.value,
registry,
snapshotEngine,
);
},
hover: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
hover: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const hoverCommand = command as Extract<BrowserAutomationCommand, { command: "hover" }>;
return executeHover(
requestId,
workspaceId,
hoverCommand.args.browserId ?? request.browserId,
hoverCommand.args.browserId,
hoverCommand.args.ref,
registry,
snapshotEngine,
);
},
drag: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => {
drag: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const dragCommand = command as Extract<BrowserAutomationCommand, { command: "drag" }>;
return executeDrag(
requestId,
workspaceId,
dragCommand.args.browserId ?? request.browserId,
dragCommand.args.browserId,
dragCommand.args.sourceRef,
dragCommand.args.targetRef,
registry,
snapshotEngine,
);
},
logs: ({ request, command, requestId, workspaceId, registry }) => {
logs: ({ command, requestId, workspaceId, registry }) => {
const logsCommand = command as Extract<BrowserAutomationCommand, { command: "logs" }>;
return executeLogs(
requestId,
workspaceId,
logsCommand.args.browserId ?? request.browserId,
logsCommand.args.browserId,
logsCommand.args.maxEntries,
registry,
);
},
storage: ({ request, command, requestId, workspaceId, registry }) => {
storage: ({ command, requestId, workspaceId, registry }) => {
const storageCommand = command as Extract<BrowserAutomationCommand, { command: "storage" }>;
return executeStorage(
requestId,
workspaceId,
storageCommand.args.browserId ?? request.browserId,
registry,
);
return executeStorage(requestId, workspaceId, storageCommand.args.browserId, registry);
},
environment: ({ request, command, requestId, workspaceId, registry }) => {
environment: ({ command, requestId, workspaceId, registry }) => {
const environmentCommand = command as Extract<
BrowserAutomationCommand,
{ command: "environment" }
@@ -496,7 +452,7 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
return executeEnvironment(
requestId,
workspaceId,
environmentCommand.args.browserId ?? request.browserId,
environmentCommand.args.browserId,
{
viewport: environmentCommand.args.viewport,
geolocation: environmentCommand.args.geolocation,
@@ -504,7 +460,7 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
registry,
);
},
set_background: ({ request, command, requestId, workspaceId, registry }) => {
set_background: ({ command, requestId, workspaceId, registry }) => {
const setBackgroundCommand = command as Extract<
BrowserAutomationCommand,
{ command: "set_background" }
@@ -512,7 +468,7 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
return executeSetBackground(
requestId,
workspaceId,
setBackgroundCommand.args.browserId ?? request.browserId,
setBackgroundCommand.args.browserId,
setBackgroundCommand.args.color,
registry,
);
@@ -555,7 +511,7 @@ function executeListTabs(
function executePageInfo(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
registry: BrowserRegistry,
): AutomationCommandPayload {
const target = resolveTabTarget({
@@ -586,7 +542,7 @@ function executePageInfo(
async function executeSnapshot(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
): Promise<AutomationCommandPayload> {
@@ -624,7 +580,7 @@ async function executeSnapshot(
async function executeClick(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
ref: string,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
@@ -647,7 +603,7 @@ async function executeClick(
async function executeFill(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
ref: string,
value: string,
registry: BrowserRegistry,
@@ -672,7 +628,7 @@ async function executeFill(
async function executeFocus(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
ref: string,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
@@ -695,7 +651,7 @@ async function executeFocus(
async function executeClear(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
ref: string,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
@@ -718,7 +674,7 @@ async function executeClear(
async function executeCheck(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
ref: string,
checked: boolean,
registry: BrowserRegistry,
@@ -747,7 +703,7 @@ async function executeCheck(
async function executeSelect(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
ref: string,
value: string,
registry: BrowserRegistry,
@@ -776,7 +732,7 @@ async function executeSelect(
async function executeHover(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
ref: string,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
@@ -799,7 +755,7 @@ async function executeHover(
async function executeDrag(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
sourceRef: string,
targetRef: string,
registry: BrowserRegistry,
@@ -828,7 +784,7 @@ async function executeDrag(
async function executeLogs(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
maxEntries: number,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
@@ -855,7 +811,7 @@ async function executeLogs(
async function executeStorage(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
@@ -882,7 +838,7 @@ async function executeStorage(
async function executeEnvironment(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
environment: {
viewport?: { width: number; height: number; deviceScaleFactor?: number };
geolocation?: { latitude: number; longitude: number; accuracy?: number };
@@ -941,7 +897,7 @@ function staleRefFailure(requestId: string, ref: string): FailurePayload {
async function executeWait(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
condition: { text?: string; url?: string; timeoutMs?: number },
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
@@ -999,7 +955,7 @@ async function executeWait(
async function executeSetBackground(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
color: string,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
@@ -1032,7 +988,7 @@ function buildSetBackgroundScript(color: string): string {
async function executeType(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
ref: string | undefined,
text: string,
registry: BrowserRegistry,
@@ -1061,7 +1017,7 @@ async function executeType(
async function executeKeypress(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
ref: string | undefined,
key: string,
registry: BrowserRegistry,
@@ -1090,7 +1046,7 @@ async function executeKeypress(
async function executeNavigate(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
url: string,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
@@ -1112,7 +1068,7 @@ async function executeNavigate(
function executeNavigationAction(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
action: "back" | "forward" | "reload",
registry: BrowserRegistry,
): AutomationCommandPayload {
@@ -1135,7 +1091,7 @@ function executeNavigationAction(
async function executeScreenshot(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
@@ -1212,7 +1168,7 @@ async function getCdpLayoutMetrics(contents: TabContents): Promise<{
async function executeFullPageScreenshot(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
@@ -1260,7 +1216,7 @@ async function executeFullPageScreenshot(
async function executePdf(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
options: { landscape?: boolean; printBackground: boolean },
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
@@ -1290,7 +1246,7 @@ async function executePdf(
async function executeDownload(
requestId: string,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
input: { url: string; fileName?: string },
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
@@ -1331,7 +1287,7 @@ async function executeUpload(
requestId: string,
cwd: string | undefined,
workspaceId: string | undefined,
browserId: string | undefined,
browserId: string,
input: { ref: string; filePaths: string[] },
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
@@ -1573,41 +1529,22 @@ const VIEWPORT_SCRIPT = String.raw`(() => JSON.stringify({
function resolveTabTarget(input: {
requestId: string;
workspaceId: string | undefined;
browserId: string | undefined;
browserId: string;
registry: BrowserRegistry;
}): ResolvedTabTarget | FailurePayload {
const { requestId, workspaceId, browserId, registry } = input;
let contents: TabContents | null;
let resolvedBrowserId: string;
if (workspaceId && registry.getBrowserWorkspaceId(browserId) !== workspaceId) {
return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`);
}
if (browserId) {
if (workspaceId && registry.getBrowserWorkspaceId(browserId) !== workspaceId) {
return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`);
}
contents = registry.getTabContents(browserId);
resolvedBrowserId = browserId;
if (!contents) {
return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`);
}
} else {
if (!workspaceId) {
return fail(requestId, "browser_no_tab", "No active browser tab in workspace");
}
contents = registry.getWorkspaceActiveTabContents(workspaceId);
const activeId = registry.getWorkspaceActiveBrowserId(workspaceId);
if (!contents || !activeId) {
return fail(requestId, "browser_no_tab", "No active browser tab in workspace");
}
resolvedBrowserId = activeId;
const contents = registry.getTabContents(browserId);
if (!contents) {
return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`);
}
if (contents.isDestroyed()) {
return fail(
requestId,
"browser_tab_closed",
`Browser tab ${resolvedBrowserId} has been closed`,
);
return fail(requestId, "browser_tab_closed", `Browser tab ${browserId} has been closed`);
}
return { browserId: resolvedBrowserId, contents };
return { browserId, contents };
}
packages/desktop/src/features/browser-webviews/index.ts
-21
View File
@@ -75,17 +75,6 @@ export function getWorkspaceActivePaseoBrowserId(workspaceId: string): string |
return browserRegistry.getWorkspaceActiveBrowserId(workspaceId);
}
export function setAgentActivePaseoBrowserId(input: {
agentId: string;
browserId: string | null;
}): void {
browserRegistry.setAgentActiveBrowser(input);
}
export function getAgentActivePaseoBrowserId(agentId: string): string | null {
return browserRegistry.getAgentActiveBrowserId(agentId);
}
export function getPaseoBrowserWebContents(browserId: string): WebContents | null {
const contentsId = browserRegistry.getWebContentsIdForBrowser(browserId);
if (contentsId === null) {
@@ -99,16 +88,6 @@ export function getPaseoBrowserWebContents(browserId: string): WebContents | nul
return null;
}
export function getWorkspaceActivePaseoBrowserWebContents(workspaceId: string): WebContents | null {
const activeBrowserId = getWorkspaceActivePaseoBrowserId(workspaceId);
return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null;
}
export function getAgentActivePaseoBrowserWebContents(agentId: string): WebContents | null {
const activeBrowserId = getAgentActivePaseoBrowserId(agentId);
return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null;
}
export function getMostRecentWorkspaceActivePaseoBrowserWebContents(): WebContents | null {
const browserId = browserRegistry.getMostRecentWorkspaceActiveBrowserId();
return browserId ? getPaseoBrowserWebContents(browserId) : null;
packages/desktop/src/features/browser-webviews/registry.test.ts
-2
View File
@@ -8,7 +8,6 @@ describe("PaseoBrowserWebviewRegistry", () => {
registry.registerWebContents({ webContentsId: 1, browserId: "browser-a" });
registry.registerWorkspace({ browserId: "browser-a", workspaceId: "workspace-a" });
registry.setWorkspaceActiveBrowser({ workspaceId: "workspace-a", browserId: "browser-a" });
registry.setAgentActiveBrowser({ agentId: "agent-a", browserId: "browser-a" });
registry.registerWebContents({ webContentsId: 2, browserId: "browser-a" });
expect(registry.getBrowserIdForWebContents(1)).toBeNull();
@@ -16,7 +15,6 @@ describe("PaseoBrowserWebviewRegistry", () => {
expect(registry.getWebContentsIdForBrowser("browser-a")).toBe(2);
expect(registry.getWorkspaceId("browser-a")).toBe("workspace-a");
expect(registry.getWorkspaceActiveBrowserId("workspace-a")).toBe("browser-a");
expect(registry.getAgentActiveBrowserId("agent-a")).toBe("browser-a");
});
it("ignores stale destroy events after a duplicate browserId moved", () => {
packages/desktop/src/features/browser-webviews/registry.ts
-19
View File
@@ -8,7 +8,6 @@ export class PaseoBrowserWebviewRegistry {
private readonly webContentsIdsByBrowserId = new Map<string, number>();
private readonly workspaceIdsByBrowserId = new Map<string, string>();
private readonly activeBrowserIdsByWorkspaceId = new Map<string, string>();
private readonly activeBrowserIdsByAgentId = new Map<string, string>();
public registerWebContents(input: { webContentsId: number; browserId: string }): void {
const previousWebContentsId = this.webContentsIdsByBrowserId.get(input.browserId) ?? null;
@@ -80,29 +79,11 @@ export class PaseoBrowserWebviewRegistry {
return Array.from(this.activeBrowserIdsByWorkspaceId.values()).at(-1) ?? null;
}
public setAgentActiveBrowser(input: { agentId: string; browserId: string | null }): void {
if (input.browserId) {
this.activeBrowserIdsByAgentId.delete(input.agentId);
this.activeBrowserIdsByAgentId.set(input.agentId, input.browserId);
return;
}
this.activeBrowserIdsByAgentId.delete(input.agentId);
}
public getAgentActiveBrowserId(agentId: string): string | null {
return this.activeBrowserIdsByAgentId.get(agentId) ?? null;
}
private deleteActiveBrowserReferences(browserId: string): void {
for (const [workspaceId, activeBrowserId] of this.activeBrowserIdsByWorkspaceId) {
if (activeBrowserId === browserId) {
this.activeBrowserIdsByWorkspaceId.delete(workspaceId);
}
}
for (const [agentId, activeBrowserId] of this.activeBrowserIdsByAgentId) {
if (activeBrowserId === browserId) {
this.activeBrowserIdsByAgentId.delete(agentId);
}
}
}
}
packages/desktop/src/main.ts
-22
View File
@@ -56,7 +56,6 @@ import {
registerBrowserWebviewNavigationGuards,
registerPaseoBrowserWorkspace,
registerPaseoBrowserWebContents,
setAgentActivePaseoBrowserId,
setWorkspaceActivePaseoBrowserId,
} from "./features/browser-webviews/index.js";
import { parseOpenProjectPathFromArgv } from "./open-project-routing.js";
@@ -144,20 +143,6 @@ function readActiveBrowserInput(
return { workspaceId: record.workspaceId.trim(), browserId: browserId || null };
}
function readAgentActiveBrowserInput(
input: unknown,
): { agentId: string; browserId: string | null } | null {
if (typeof input !== "object" || input === null || Array.isArray(input)) {
return null;
}
const record = input as Record<string, unknown>;
if (typeof record.agentId !== "string" || record.agentId.trim().length === 0) {
return null;
}
const browserId = typeof record.browserId === "string" ? record.browserId.trim() : null;
return { agentId: record.agentId.trim(), browserId: browserId || null };
}
const pendingBrowserWebviewIds: string[] = [];
function isBrowserRefreshInput(input: Electron.Input): boolean {
@@ -350,13 +335,6 @@ ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput:
}
});
ipcMain.handle("paseo:browser:set-agent-active-browser", (_event, rawInput: unknown) => {
const input = readAgentActiveBrowserInput(rawInput);
if (input) {
setAgentActivePaseoBrowserId(input);
}
});
ipcMain.handle("paseo:browser:open-devtools", (_event, browserId: unknown) => {
if (typeof browserId !== "string" || browserId.trim().length === 0) {
const result = {
packages/desktop/src/preload.ts
-2
View File
@@ -78,8 +78,6 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
ipcRenderer.invoke("paseo:browser:register-workspace-browser", input),
setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) =>
ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input),
setAgentActiveBrowser: (input: { agentId: string; browserId: string | null }) =>
ipcRenderer.invoke("paseo:browser:set-agent-active-browser", input),
openDevTools: (browserId: string) =>
ipcRenderer.invoke("paseo:browser:open-devtools", browserId),
clearPartition: (browserId: string) =>
packages/protocol/src/browser-automation/rpc-schemas.test.ts
+181 -725
View File
@@ -5,96 +5,215 @@ import {
BrowserAutomationExecuteResponseSchema,
} from "./rpc-schemas.js";
describe("browser automation execute RPC schemas", () => {
test("rejects navigate and download commands for non-http URLs", () => {
for (const command of ["navigate", "download"] as const) {
expect(() =>
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: `req-${command}`,
command: {
command,
args: { url: "file:///tmp/secret.txt" },
},
}),
).toThrow();
}
});
const BROWSER_ID = "11111111-1111-4111-8111-111111111111";
const FALLBACK_BROWSER_ID = "1777777777777-abcdef";
const BROWSER_ID_MESSAGE =
"browserId must be a real id returned by browser_new_tab or browser_list_tabs";
const WAIT_CONDITION_MESSAGE = "browser_wait requires exactly one of text or url";
test("parses list tabs requests with top-level correlation and typed command args", () => {
describe("browser automation execute RPC schemas", () => {
test("list tabs reads workspace from the request envelope", () => {
const parsed = BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-1",
requestId: "req-list-tabs",
workspaceId: "workspace-1",
command: {
command: "list_tabs",
args: { workspaceId: "workspace-1" },
},
command: { command: "list_tabs", args: {} },
});
expect(parsed).toEqual({
type: "browser.automation.execute.request",
requestId: "req-1",
requestId: "req-list-tabs",
workspaceId: "workspace-1",
command: {
command: "list_tabs",
args: { workspaceId: "workspace-1" },
},
command: { command: "list_tabs", args: {} },
});
});
test("parses new tab requests and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-new-tab",
workspaceId: "workspace-1",
command: {
command: "new_tab",
args: { workspaceId: "workspace-1", url: "https://example.com" },
},
}).command,
).toEqual({
command: "new_tab",
args: { workspaceId: "workspace-1", url: "https://example.com" },
test("new tab reads workspace from the request envelope", () => {
const parsed = BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-new-tab",
workspaceId: "workspace-1",
command: { command: "new_tab", args: { url: "https://example.com" } },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-new-tab",
ok: true,
result: {
command: "new_tab",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
},
expect(parsed.command).toEqual({
command: "new_tab",
args: { url: "https://example.com" },
});
});
test("tab commands require a browser id from browser_new_tab or browser_list_tabs", () => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId: "req-snapshot",
workspaceId: "workspace-1",
command: { command: "snapshot", args: {} },
});
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] },
});
});
test("tab commands reject hallucinated browser ids", () => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId: "req-page-info",
workspaceId: "workspace-1",
command: { command: "page_info", args: { browserId: "default" } },
});
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] },
});
});
test("tab commands parse browser ids produced by the fallback generator", () => {
const parsed = BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-page-info",
workspaceId: "workspace-1",
command: { command: "page_info", args: { browserId: FALLBACK_BROWSER_ID } },
});
expect(parsed.command).toEqual({
command: "page_info",
args: { browserId: FALLBACK_BROWSER_ID },
});
});
test("requests reject browser id in the envelope", () => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId: "req-click",
workspaceId: "workspace-1",
browserId: BROWSER_ID,
command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } },
});
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: 'Unrecognized key: "browserId"' })] },
});
});
test("tab commands reject workspace id in command args", () => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId: "req-click",
workspaceId: "workspace-1",
command: {
command: "click",
args: { workspaceId: "workspace-1", browserId: BROWSER_ID, ref: "@e1" },
},
});
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: 'Unrecognized key: "workspaceId"' })] },
});
});
test("wait rejects calls without exactly one condition", () => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId: "req-wait",
command: { command: "wait", args: { browserId: BROWSER_ID } },
});
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: WAIT_CONDITION_MESSAGE })] },
});
});
test("wait rejects calls with both text and url conditions", () => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId: "req-wait",
command: {
command: "wait",
args: { browserId: BROWSER_ID, text: "Ready", url: "/ready" },
},
});
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: WAIT_CONDITION_MESSAGE })] },
});
});
test("wait accepts one text condition", () => {
const parsed = BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-wait",
command: {
command: "wait",
args: { browserId: BROWSER_ID, text: "Ready", timeoutMs: 1000 },
},
});
expect(parsed.command).toEqual({
command: "wait",
args: { browserId: BROWSER_ID, text: "Ready", timeoutMs: 1000 },
});
});
test("navigate rejects non-http URLs at the protocol boundary", () => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId: "req-navigate",
command: {
command: "navigate",
args: { browserId: BROWSER_ID, url: "file:///tmp/secret.txt" },
},
});
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: "URL must use http or https" })] },
});
});
test("new tab responses declare the generated browser id shape", () => {
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-new-tab",
ok: true,
result: {
command: "new_tab",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
},
}).payload,
).toEqual({
},
});
expect(parsed.payload).toEqual({
requestId: "req-new-tab",
ok: true,
result: {
command: "new_tab",
browserId: "browser-1",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
},
});
});
test("parses page info responses with result data under payload", () => {
const parsed = BrowserAutomationExecuteResponseSchema.parse({
test("responses reject hallucinated browser ids", () => {
const parsed = BrowserAutomationExecuteResponseSchema.safeParse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
requestId: "req-page-info",
ok: true,
result: {
command: "page_info",
tab: {
browserId: "browser-1",
browserId: "default",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
@@ -103,672 +222,9 @@ describe("browser automation execute RPC schemas", () => {
},
});
expect(parsed.payload).toEqual({
requestId: "req-1",
ok: true,
result: {
command: "page_info",
tab: {
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
isActive: false,
isLoading: false,
},
},
});
});
test("parses snapshot requests and ref-bearing snapshot responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-snapshot",
workspaceId: "workspace-1",
command: {
command: "snapshot",
args: { workspaceId: "workspace-1", browserId: "browser-1" },
},
}),
).toEqual({
type: "browser.automation.execute.request",
requestId: "req-snapshot",
workspaceId: "workspace-1",
command: {
command: "snapshot",
args: { workspaceId: "workspace-1", browserId: "browser-1" },
},
});
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-snapshot",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name", type: "text" },
},
],
},
},
});
expect(parsed.payload).toEqual({
requestId: "req-snapshot",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name", type: "text" },
},
],
},
});
});
test("parses click and fill ref commands", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-click",
command: {
command: "click",
args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" },
},
}).command,
).toEqual({
command: "click",
args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" },
});
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-fill",
command: {
command: "fill",
args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" },
},
}).command,
).toEqual({
command: "fill",
args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-fill",
ok: true,
result: { command: "fill", browserId: "browser-1", ref: "@e2" },
},
}).payload,
).toEqual({
requestId: "req-fill",
ok: true,
result: { command: "fill", browserId: "browser-1", ref: "@e2" },
});
});
test("parses wait text commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-wait",
command: {
command: "wait",
args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 },
},
}).command,
).toEqual({
command: "wait",
args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-wait",
ok: true,
result: { command: "wait", browserId: "browser-1", matched: "text" },
},
}).payload,
).toEqual({
requestId: "req-wait",
ok: true,
result: { command: "wait", browserId: "browser-1", matched: "text" },
});
});
test("parses set background commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-bg",
command: {
command: "set_background",
args: { workspaceId: "workspace-1", color: "red" },
},
}).command,
).toEqual({
command: "set_background",
args: { workspaceId: "workspace-1", color: "red" },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-bg",
ok: true,
result: { command: "set_background", browserId: "browser-1", color: "red" },
},
}).payload,
).toEqual({
requestId: "req-bg",
ok: true,
result: { command: "set_background", browserId: "browser-1", color: "red" },
});
});
test("parses type and keypress commands", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-type",
command: { command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } },
}).command,
).toEqual({ command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-keypress",
command: { command: "keypress", args: { browserId: "browser-1", key: "Enter" } },
}).command,
).toEqual({ command: "keypress", args: { browserId: "browser-1", key: "Enter" } });
});
test("parses navigation commands", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-nav",
command: {
command: "navigate",
args: { browserId: "browser-1", url: "https://example.com/next" },
},
}).command,
).toEqual({
command: "navigate",
args: { browserId: "browser-1", url: "https://example.com/next" },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-nav",
ok: true,
result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" },
},
}).payload,
).toEqual({
requestId: "req-nav",
ok: true,
result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" },
});
});
test("parses screenshot responses", () => {
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-shot",
ok: true,
result: {
command: "screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 100,
height: 50,
},
},
}).payload,
).toEqual({
requestId: "req-shot",
ok: true,
result: {
command: "screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 100,
height: 50,
},
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-shot-no-frame",
ok: false,
error: {
code: "screenshot_no_frame",
message:
"The browser tab has no painted frame. Focus the tab in the app, then try again.",
retryable: false,
},
},
}).payload,
).toEqual({
requestId: "req-shot-no-frame",
ok: false,
error: {
code: "screenshot_no_frame",
message: "The browser tab has no painted frame. Focus the tab in the app, then try again.",
retryable: false,
},
});
});
test("parses form control commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-focus",
command: { command: "focus", args: { browserId: "browser-1", ref: "@e1" } },
}).command,
).toEqual({ command: "focus", args: { browserId: "browser-1", ref: "@e1" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-clear",
command: { command: "clear", args: { browserId: "browser-1", ref: "@e1" } },
}).command,
).toEqual({ command: "clear", args: { browserId: "browser-1", ref: "@e1" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-check",
command: { command: "check", args: { browserId: "browser-1", ref: "@e2" } },
}).command,
).toEqual({ command: "check", args: { browserId: "browser-1", ref: "@e2", checked: true } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-select",
command: { command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } },
}).command,
).toEqual({ command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-hover",
command: { command: "hover", args: { browserId: "browser-1", ref: "@e4" } },
}).command,
).toEqual({ command: "hover", args: { browserId: "browser-1", ref: "@e4" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-drag",
command: {
command: "drag",
args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" },
},
}).command,
).toEqual({
command: "drag",
args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-select",
ok: true,
result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" },
},
}).payload,
).toEqual({
requestId: "req-select",
ok: true,
result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" },
});
});
test("parses browser log commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-logs",
command: { command: "logs", args: { browserId: "browser-1" } },
}).command,
).toEqual({ command: "logs", args: { browserId: "browser-1", maxEntries: 50 } });
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-logs",
ok: true,
result: {
command: "logs",
browserId: "browser-1",
console: [{ level: "info", message: "ready", timestamp: 10 }],
network: [
{
url: "https://example.com/app.js",
type: "script",
startTime: 1,
duration: 2,
},
],
},
},
}).payload,
).toEqual({
requestId: "req-logs",
ok: true,
result: {
command: "logs",
browserId: "browser-1",
console: [{ level: "info", message: "ready", timestamp: 10 }],
network: [
{
url: "https://example.com/app.js",
type: "script",
startTime: 1,
duration: 2,
},
],
},
});
});
test("parses browser storage commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-storage",
command: { command: "storage", args: { browserId: "browser-1" } },
}).command,
).toEqual({ command: "storage", args: { browserId: "browser-1" } });
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-storage",
ok: true,
result: {
command: "storage",
browserId: "browser-1",
url: "https://example.com",
cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
},
}).payload,
).toEqual({
requestId: "req-storage",
ok: true,
result: {
command: "storage",
browserId: "browser-1",
url: "https://example.com",
cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
});
});
test("parses browser environment commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-environment",
command: {
command: "environment",
args: {
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
}).command,
).toEqual({
command: "environment",
args: {
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-environment",
ok: true,
result: {
command: "environment",
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
}).payload,
).toEqual({
requestId: "req-environment",
ok: true,
result: {
command: "environment",
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
});
});
test("parses browser full-page screenshot and PDF commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-full-page",
command: { command: "full_page_screenshot", args: { browserId: "browser-1" } },
}).command,
).toEqual({ command: "full_page_screenshot", args: { browserId: "browser-1" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-pdf",
command: { command: "pdf", args: { browserId: "browser-1" } },
}).command,
).toEqual({ command: "pdf", args: { browserId: "browser-1", printBackground: true } });
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-full-page",
ok: true,
result: {
command: "full_page_screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 390,
height: 1200,
},
},
}).payload.result,
).toEqual({
command: "full_page_screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 390,
height: 1200,
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-pdf",
ok: true,
result: {
command: "pdf",
browserId: "browser-1",
mimeType: "application/pdf",
dataBase64: "JVBERi0xLjQ=",
},
},
}).payload.result,
).toEqual({
command: "pdf",
browserId: "browser-1",
mimeType: "application/pdf",
dataBase64: "JVBERi0xLjQ=",
});
});
test("parses browser download and upload commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-download",
command: {
command: "download",
args: { browserId: "browser-1", url: "https://example.com/file.txt" },
},
}).command,
).toEqual({
command: "download",
args: { browserId: "browser-1", url: "https://example.com/file.txt" },
});
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-upload",
command: {
command: "upload",
args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] },
},
}).command,
).toEqual({
command: "upload",
args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-download",
ok: true,
result: {
command: "download",
browserId: "browser-1",
url: "https://example.com/file.txt",
filePath: "/tmp/file.txt",
totalBytes: 5,
state: "completed",
},
},
}).payload.result,
).toEqual({
command: "download",
browserId: "browser-1",
url: "https://example.com/file.txt",
filePath: "/tmp/file.txt",
totalBytes: 5,
state: "completed",
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-upload",
ok: true,
result: {
command: "upload",
browserId: "browser-1",
ref: "@e1",
filePaths: ["/tmp/file.txt"],
},
},
}).payload.result,
).toEqual({
command: "upload",
browserId: "browser-1",
ref: "@e1",
filePaths: ["/tmp/file.txt"],
});
});
test("parses stable model-actionable error responses", () => {
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: false,
error: {
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
},
},
});
expect(parsed.payload).toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
retryable: false,
},
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] },
});
});
});
packages/protocol/src/browser-automation/rpc-schemas.ts
+69 -58
View File
@@ -3,7 +3,6 @@ import { z } from "zod";
export const BrowserAutomationErrorCodeSchema = z.enum([
"browser_disabled",
"browser_no_desktop",
"browser_no_tab",
"browser_tab_not_found",
"browser_tab_closed",
"browser_timeout",
@@ -14,10 +13,23 @@ export const BrowserAutomationErrorCodeSchema = z.enum([
"browser_unknown_error",
]);
const BrowserAutomationTabTargetSchema = z.object({
workspaceId: z.string().min(1).optional(),
browserId: z.string().min(1).optional(),
});
const BROWSER_AUTOMATION_BROWSER_ID_PATTERN =
/^(?:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|\d{13,}-[0-9a-f]+)$/i;
const BROWSER_AUTOMATION_BROWSER_ID_MESSAGE =
"browserId must be a real id returned by browser_new_tab or browser_list_tabs";
const BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE =
"browser_wait requires exactly one of text or url";
export const BrowserAutomationBrowserIdSchema = z
.string({ error: () => BROWSER_AUTOMATION_BROWSER_ID_MESSAGE })
.min(1, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE)
.regex(BROWSER_AUTOMATION_BROWSER_ID_PATTERN, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE);
const BrowserAutomationTabTargetSchema = z
.object({
browserId: BrowserAutomationBrowserIdSchema,
})
.strict();
const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/);
const BrowserAutomationHttpUrlSchema = z
@@ -30,31 +42,27 @@ const BrowserAutomationHttpUrlSchema = z
export const BrowserAutomationListTabsCommandSchema = z.object({
command: z.literal("list_tabs"),
args: z
.object({
workspaceId: z.string().min(1).optional(),
})
.default({}),
args: z.object({}).strict().default({}),
});
export const BrowserAutomationNewTabCommandSchema = z.object({
command: z.literal("new_tab"),
args: z
.object({
workspaceId: z.string().min(1).optional(),
url: BrowserAutomationHttpUrlSchema.optional(),
})
.strict()
.default({}),
});
export const BrowserAutomationPageInfoCommandSchema = z.object({
command: z.literal("page_info"),
args: BrowserAutomationTabTargetSchema.default({}),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationSnapshotCommandSchema = z.object({
command: z.literal("snapshot"),
args: BrowserAutomationTabTargetSchema.default({}),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationClickCommandSchema = z.object({
@@ -78,6 +86,8 @@ export const BrowserAutomationWaitCommandSchema = z.object({
text: z.string().min(1).optional(),
url: z.string().min(1).optional(),
timeoutMs: z.number().int().positive().max(30_000).optional(),
}).refine((args) => Number(Boolean(args.text)) + Number(Boolean(args.url)) === 1, {
message: BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE,
}),
});
@@ -106,27 +116,27 @@ export const BrowserAutomationNavigateCommandSchema = z.object({
export const BrowserAutomationBackCommandSchema = z.object({
command: z.literal("back"),
args: BrowserAutomationTabTargetSchema.default({}),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationForwardCommandSchema = z.object({
command: z.literal("forward"),
args: BrowserAutomationTabTargetSchema.default({}),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationReloadCommandSchema = z.object({
command: z.literal("reload"),
args: BrowserAutomationTabTargetSchema.default({}),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationScreenshotCommandSchema = z.object({
command: z.literal("screenshot"),
args: BrowserAutomationTabTargetSchema.default({}),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({
command: z.literal("full_page_screenshot"),
args: BrowserAutomationTabTargetSchema.default({}),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationPdfCommandSchema = z.object({
@@ -134,7 +144,7 @@ export const BrowserAutomationPdfCommandSchema = z.object({
args: BrowserAutomationTabTargetSchema.extend({
landscape: z.boolean().optional(),
printBackground: z.boolean().default(true),
}).default({ printBackground: true }),
}),
});
export const BrowserAutomationDownloadCommandSchema = z.object({
@@ -202,12 +212,12 @@ export const BrowserAutomationLogsCommandSchema = z.object({
command: z.literal("logs"),
args: BrowserAutomationTabTargetSchema.extend({
maxEntries: z.number().int().positive().max(200).default(50),
}).default({ maxEntries: 50 }),
}),
});
export const BrowserAutomationStorageCommandSchema = z.object({
command: z.literal("storage"),
args: BrowserAutomationTabTargetSchema.default({}),
args: BrowserAutomationTabTargetSchema,
});
const BrowserAutomationViewportInputSchema = z.object({
@@ -227,7 +237,7 @@ export const BrowserAutomationEnvironmentCommandSchema = z.object({
args: BrowserAutomationTabTargetSchema.extend({
viewport: BrowserAutomationViewportInputSchema.optional(),
geolocation: BrowserAutomationGeolocationInputSchema.optional(),
}).default({}),
}),
});
export const BrowserAutomationSetBackgroundCommandSchema = z.object({
@@ -269,7 +279,7 @@ export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
]);
export const BrowserAutomationTabInfoSchema = z.object({
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
workspaceId: z.string().min(1).optional(),
url: z.string(),
title: z.string(),
@@ -286,7 +296,7 @@ export const BrowserAutomationListTabsResultSchema = z.object({
export const BrowserAutomationNewTabResultSchema = z.object({
command: z.literal("new_tab"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
workspaceId: z.string().min(1),
url: z.string().min(1),
});
@@ -307,7 +317,7 @@ export const BrowserAutomationSnapshotElementSchema = z.object({
export const BrowserAutomationSnapshotResultSchema = z.object({
command: z.literal("snapshot"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
workspaceId: z.string().min(1).optional(),
url: z.string(),
title: z.string(),
@@ -316,59 +326,59 @@ export const BrowserAutomationSnapshotResultSchema = z.object({
export const BrowserAutomationClickResultSchema = z.object({
command: z.literal("click"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationFillResultSchema = z.object({
command: z.literal("fill"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationWaitResultSchema = z.object({
command: z.literal("wait"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
matched: z.enum(["text", "url"]),
});
export const BrowserAutomationTypeResultSchema = z.object({
command: z.literal("type"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema.optional(),
});
export const BrowserAutomationKeypressResultSchema = z.object({
command: z.literal("keypress"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
key: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
});
export const BrowserAutomationNavigateResultSchema = z.object({
command: z.literal("navigate"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
url: z.string().min(1),
});
export const BrowserAutomationBackResultSchema = z.object({
command: z.literal("back"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
});
export const BrowserAutomationForwardResultSchema = z.object({
command: z.literal("forward"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
});
export const BrowserAutomationReloadResultSchema = z.object({
command: z.literal("reload"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
});
export const BrowserAutomationScreenshotResultSchema = z.object({
command: z.literal("screenshot"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
mimeType: z.literal("image/png"),
dataBase64: z.string().min(1),
width: z.number().int().nonnegative(),
@@ -377,7 +387,7 @@ export const BrowserAutomationScreenshotResultSchema = z.object({
export const BrowserAutomationFullPageScreenshotResultSchema = z.object({
command: z.literal("full_page_screenshot"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
mimeType: z.literal("image/png"),
dataBase64: z.string().min(1),
width: z.number().int().nonnegative(),
@@ -386,14 +396,14 @@ export const BrowserAutomationFullPageScreenshotResultSchema = z.object({
export const BrowserAutomationPdfResultSchema = z.object({
command: z.literal("pdf"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
mimeType: z.literal("application/pdf"),
dataBase64: z.string().min(1),
});
export const BrowserAutomationDownloadResultSchema = z.object({
command: z.literal("download"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
url: z.string().min(1),
filePath: z.string().min(1),
totalBytes: z.number().int().nonnegative().optional(),
@@ -402,46 +412,46 @@ export const BrowserAutomationDownloadResultSchema = z.object({
export const BrowserAutomationUploadResultSchema = z.object({
command: z.literal("upload"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
filePaths: z.array(z.string().min(1)).min(1),
});
export const BrowserAutomationFocusResultSchema = z.object({
command: z.literal("focus"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationClearResultSchema = z.object({
command: z.literal("clear"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationCheckResultSchema = z.object({
command: z.literal("check"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
checked: z.boolean(),
});
export const BrowserAutomationSelectResultSchema = z.object({
command: z.literal("select"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
value: z.string(),
});
export const BrowserAutomationHoverResultSchema = z.object({
command: z.literal("hover"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationDragResultSchema = z.object({
command: z.literal("drag"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
sourceRef: BrowserAutomationRefSchema,
targetRef: BrowserAutomationRefSchema,
});
@@ -466,7 +476,7 @@ export const BrowserAutomationNetworkLogEntrySchema = z.object({
export const BrowserAutomationLogsResultSchema = z.object({
command: z.literal("logs"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
console: z.array(BrowserAutomationConsoleLogEntrySchema),
network: z.array(BrowserAutomationNetworkLogEntrySchema),
});
@@ -488,7 +498,7 @@ export const BrowserAutomationStorageEntrySchema = z.object({
export const BrowserAutomationStorageResultSchema = z.object({
command: z.literal("storage"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
url: z.string(),
cookies: z.array(BrowserAutomationCookieEntrySchema),
localStorage: z.array(BrowserAutomationStorageEntrySchema),
@@ -509,14 +519,14 @@ export const BrowserAutomationGeolocationResultSchema = z.object({
export const BrowserAutomationEnvironmentResultSchema = z.object({
command: z.literal("environment"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
viewport: BrowserAutomationViewportResultSchema,
geolocation: BrowserAutomationGeolocationResultSchema.optional(),
});
export const BrowserAutomationSetBackgroundResultSchema = z.object({
command: z.literal("set_background"),
browserId: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema,
color: z.string().min(1),
});
@@ -557,15 +567,16 @@ export const BrowserAutomationErrorSchema = z.object({
retryable: z.boolean().default(false),
});
export const BrowserAutomationExecuteRequestSchema = z.object({
type: z.literal("browser.automation.execute.request"),
requestId: z.string().min(1),
agentId: z.string().min(1).optional(),
cwd: z.string().min(1).optional(),
workspaceId: z.string().min(1).optional(),
browserId: z.string().min(1).optional(),
command: BrowserAutomationCommandSchema,
});
export const BrowserAutomationExecuteRequestSchema = z
.object({
type: z.literal("browser.automation.execute.request"),
requestId: z.string().min(1),
agentId: z.string().min(1).optional(),
cwd: z.string().min(1).optional(),
workspaceId: z.string().min(1).optional(),
command: BrowserAutomationCommandSchema,
})
.strict();
export const BrowserAutomationExecuteResponseSchema = z.object({
type: z.literal("browser.automation.execute.response"),
packages/protocol/src/messages.browser-automation.test.ts
+3 -1
View File
@@ -10,6 +10,8 @@ import {
} from "./messages.js";
describe("browser automation protocol integration", () => {
const browserId = "11111111-1111-4111-8111-111111111111";
test("desktop automation capability parses in hello without narrowing old clients", () => {
expect(
WSHelloMessageSchema.parse({
@@ -39,7 +41,7 @@ describe("browser automation protocol integration", () => {
const parsed = SessionOutboundMessageSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-1",
command: { command: "page_info", args: { browserId: "browser-1" } },
command: { command: "page_info", args: { browserId } },
});
expect(parsed.type).toBe("browser.automation.execute.request");
packages/server/src/server/agent/mcp-server.test.ts
+5 -7
View File
@@ -609,8 +609,7 @@ describe("browser MCP tools", () => {
expect(execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: REPO_CWD,
workspaceId: REPO_CWD,
command: { command: "list_tabs", args: { workspaceId: REPO_CWD } },
command: { command: "list_tabs", args: {} },
});
expect(response.structuredContent).toEqual({
ok: false,
@@ -619,7 +618,7 @@ describe("browser MCP tools", () => {
message: "Browser tools are disabled.",
retryable: false,
},
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD },
context: { agentId: "agent-1", cwd: REPO_CWD },
});
});
@@ -646,19 +645,18 @@ describe("browser MCP tools", () => {
expect(execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: REPO_CWD,
workspaceId: REPO_CWD,
command: { command: "list_tabs", args: { workspaceId: REPO_CWD } },
command: { command: "list_tabs", args: {} },
});
expect(response.content).toEqual([
{
type: "text",
text: "No Paseo browser tabs are open. Call browser_new_tab to create one, then use the returned browserId or omit browserId for the active tab.",
text: "No Paseo browser tabs are open. Call browser_new_tab to create one.",
},
]);
expect(response.structuredContent).toEqual({
ok: true,
result: { command: "list_tabs", tabs: [] },
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD },
context: { agentId: "agent-1", cwd: REPO_CWD },
});
});
});
packages/server/src/server/browser-tools/broker.test.ts
+13 -11
View File
@@ -7,6 +7,8 @@ import type {
import { BrowserToolsBroker, type BrowserToolsDesktopClient } from "./broker.js";
import { StaticBrowserToolsPolicy } from "./policy.js";
const BROWSER_ID = "11111111-1111-4111-8111-111111111111";
class FakeDesktopClient implements BrowserToolsDesktopClient {
public readonly receivedRequests: BrowserAutomationExecuteRequest[] = [];
@@ -44,7 +46,7 @@ function createBroker(options: { enabled: boolean; timeoutMs?: number }): Browse
}
function pageInfoCommand(): BrowserAutomationCommand {
return { command: "page_info", args: { browserId: "browser-1" } };
return { command: "page_info", args: { browserId: BROWSER_ID } };
}
describe("BrowserToolsBroker", () => {
@@ -111,7 +113,7 @@ describe("BrowserToolsBroker", () => {
broker.registerClient(client);
const resultPromise = broker.execute({
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
command: { command: "list_tabs", args: {} },
workspaceId: "workspace-1",
});
@@ -120,7 +122,7 @@ describe("BrowserToolsBroker", () => {
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
command: { command: "list_tabs", args: {} },
},
]);
expect(broker.getPendingRequestCount()).toBe(1);
@@ -133,7 +135,7 @@ describe("BrowserToolsBroker", () => {
command: "list_tabs",
tabs: [
{
browserId: "browser-1",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
@@ -150,7 +152,7 @@ describe("BrowserToolsBroker", () => {
command: "list_tabs",
tabs: [
{
browserId: "browser-1",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
@@ -169,7 +171,7 @@ describe("BrowserToolsBroker", () => {
broker.registerClient(client);
const resultPromise = broker.execute({
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
workspaceId: "workspace-1",
});
@@ -178,7 +180,7 @@ describe("BrowserToolsBroker", () => {
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
},
]);
@@ -187,7 +189,7 @@ describe("BrowserToolsBroker", () => {
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
@@ -200,7 +202,7 @@ describe("BrowserToolsBroker", () => {
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
@@ -281,7 +283,7 @@ describe("BrowserToolsBroker", () => {
ok: false,
error: {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
message: `Browser tab ${BROWSER_ID} was not found.`,
retryable: false,
},
});
@@ -291,7 +293,7 @@ describe("BrowserToolsBroker", () => {
ok: false,
error: {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
message: `Browser tab ${BROWSER_ID} was not found.`,
retryable: false,
},
});
packages/server/src/server/browser-tools/broker.ts
-2
View File
@@ -19,7 +19,6 @@ export interface BrowserToolsExecuteInput {
agentId?: string;
cwd?: string;
workspaceId?: string;
browserId?: string;
requestId?: string;
timeoutMs?: number;
}
@@ -114,7 +113,6 @@ export class BrowserToolsBroker {
...(input.agentId ? { agentId: input.agentId } : {}),
...(input.cwd ? { cwd: input.cwd } : {}),
...(input.workspaceId ? { workspaceId: input.workspaceId } : {}),
...(input.browserId ? { browserId: input.browserId } : {}),
command: input.command,
});
packages/server/src/server/browser-tools/tools.test.ts
+237 -870
View File
File diff suppressed because it is too large Load Diff
packages/server/src/server/browser-tools/tools.ts
+95 -113
View File
@@ -1,4 +1,5 @@
import { z } from "zod";
import { BrowserAutomationBrowserIdSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { BrowserToolsBroker } from "./broker.js";
import type { BrowserToolsResponsePayload } from "./errors.js";
import type {
@@ -49,6 +50,16 @@ const BrowserToolOutputSchema = z.object({
});
const BrowserRefInputSchema = z.string().regex(/^@e\d+$/);
const BrowserWaitInputSchema = z
.object({
text: z.string().min(1).optional(),
url: z.string().min(1).optional(),
timeoutMs: z.number().int().positive().max(30_000).optional(),
browserId: BrowserAutomationBrowserIdSchema,
})
.refine((input) => Number(Boolean(input.text)) + Number(Boolean(input.url)) === 1, {
message: "browser_wait requires exactly one of text or url",
});
export function registerBrowserTools(options: RegisterBrowserToolsOptions): void {
options.registerTool(
@@ -56,7 +67,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{
title: "List browser tabs",
description:
"List open Paseo desktop browser tabs for this agent's workspace context. Use the returned browserId values exactly; do not use 'default'. If a tool omits browserId, it targets the active workspace tab.",
"List open Paseo desktop browser tabs for this agent's workspace context. Use the returned browserId values for tab-scoped browser tools.",
inputSchema: {},
outputSchema: BrowserToolOutputSchema,
},
@@ -68,7 +79,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "list_tabs",
args: context.workspaceId ? { workspaceId: context.workspaceId } : {},
args: {},
},
});
return browserToolResult({ payload, context });
@@ -80,7 +91,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{
title: "Create browser tab",
description:
"Create and focus a new Paseo desktop browser tab in this agent's workspace. Optionally pass an http(s) URL to open immediately. Returns the new browserId; use that exact ID for later calls or omit browserId to use the active tab.",
"Create and focus a new Paseo desktop browser tab in this agent's workspace. Optionally pass an http(s) URL to open immediately. Returns the browserId for later tab-scoped browser tools.",
inputSchema: {
url: z.string().url().optional(),
},
@@ -94,10 +105,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
command: {
command: "new_tab",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(url ? { url } : {}),
},
args: url ? { url } : {},
},
});
return browserToolResult({ payload, context });
@@ -109,9 +117,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{
title: "Get browser page info",
description:
"Get page info for a Paseo desktop browser tab. Omit browserId to use the active workspace tab. If you pass browserId, use a real ID returned by browser_list_tabs or browser_new_tab; never pass 'default'.",
"Get page info for a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.",
inputSchema: {
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -121,12 +129,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "page_info",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
},
},
});
@@ -139,9 +146,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{
title: "Snapshot browser page",
description:
"Return a model-readable snapshot of the active Paseo desktop browser tab for this agent's workspace. Snapshot refs like @e1 are valid until the page changes or a new snapshot is taken.",
"Return a model-readable snapshot of a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs. Snapshot refs like @e1 are valid until the page changes or a new snapshot is taken.",
inputSchema: {
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -151,12 +158,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "snapshot",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
},
},
});
@@ -172,7 +178,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"Click an element ref from the latest browser_snapshot for this agent's workspace browser tab.",
inputSchema: {
ref: BrowserRefInputSchema,
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -182,12 +188,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "click",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
ref,
},
},
@@ -205,7 +210,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
inputSchema: {
ref: BrowserRefInputSchema,
value: z.string(),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -215,12 +220,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "fill",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
ref,
value,
},
@@ -235,13 +239,8 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{
title: "Wait for browser condition",
description:
"Wait until the active Paseo desktop browser tab contains text or reaches a URL fragment.",
inputSchema: {
text: z.string().min(1).optional(),
url: z.string().min(1).optional(),
timeoutMs: z.number().int().positive().max(30_000).optional(),
browserId: z.string().min(1).optional(),
},
"Wait until a Paseo desktop browser tab contains text or reaches a URL fragment. Pass browserId from browser_new_tab or browser_list_tabs.",
inputSchema: BrowserWaitInputSchema,
outputSchema: BrowserToolOutputSchema,
},
async ({ text, url, timeoutMs, browserId }) => {
@@ -250,13 +249,12 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
...(timeoutMs ? { timeoutMs: timeoutMs + 1_000 } : {}),
command: {
command: "wait",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
...(text ? { text } : {}),
...(url ? { url } : {}),
...(timeoutMs ? { timeoutMs } : {}),
@@ -276,7 +274,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
inputSchema: {
text: z.string(),
ref: BrowserRefInputSchema.optional(),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -286,12 +284,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "type",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
...(ref ? { ref } : {}),
text,
},
@@ -310,7 +307,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
inputSchema: {
key: z.string().min(1),
ref: BrowserRefInputSchema.optional(),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -320,12 +317,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "keypress",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
...(ref ? { ref } : {}),
key,
},
@@ -339,8 +335,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"browser_navigate",
{
title: "Navigate browser",
description: "Navigate the active Paseo desktop browser tab to a URL.",
inputSchema: { url: z.string().min(1), browserId: z.string().min(1).optional() },
description:
"Navigate a Paseo desktop browser tab to a URL. Pass browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { url: z.string().min(1), browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema,
},
async ({ url, browserId }) => {
@@ -349,12 +346,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "navigate",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
url,
},
},
@@ -369,8 +365,8 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
name,
{
title: `Browser ${command}`,
description: `${command} the active Paseo desktop browser tab.`,
inputSchema: { browserId: z.string().min(1).optional() },
description: `${command} a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.`,
inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema,
},
async ({ browserId }) => {
@@ -379,12 +375,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command,
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
},
},
});
@@ -397,8 +392,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"browser_screenshot",
{
title: "Capture browser screenshot",
description: "Capture a PNG screenshot of the active Paseo desktop browser tab.",
inputSchema: { browserId: z.string().min(1).optional() },
description:
"Capture a PNG screenshot of a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema,
},
async ({ browserId }) => {
@@ -407,12 +403,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "screenshot",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
},
},
});
@@ -425,7 +420,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{
title: "Capture full-page browser screenshot",
description: "Capture a full-page PNG screenshot of a Paseo desktop browser tab.",
inputSchema: { browserId: z.string().min(1).optional() },
inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema,
},
async ({ browserId }) => {
@@ -434,12 +429,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "full_page_screenshot",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
},
},
});
@@ -451,9 +445,10 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"browser_pdf",
{
title: "Export browser page PDF",
description: "Export the current Paseo desktop browser tab as a PDF.",
description:
"Export a Paseo desktop browser tab as a PDF. Pass browserId from browser_new_tab or browser_list_tabs.",
inputSchema: {
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
landscape: z.boolean().optional(),
printBackground: z.boolean().default(true),
},
@@ -465,12 +460,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "pdf",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
...(landscape !== undefined ? { landscape } : {}),
printBackground: printBackground ?? true,
},
@@ -488,7 +482,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
inputSchema: {
url: z.string().min(1),
fileName: z.string().min(1).optional(),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -498,12 +492,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "download",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
url,
...(fileName ? { fileName } : {}),
},
@@ -522,7 +515,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
inputSchema: {
ref: BrowserRefInputSchema,
filePaths: z.array(z.string().min(1)).min(1),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -532,12 +525,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "upload",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
ref,
filePaths,
},
@@ -572,7 +564,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{
title: toolConfig.title,
description: toolConfig.description,
inputSchema: { ref: BrowserRefInputSchema, browserId: z.string().min(1).optional() },
inputSchema: { ref: BrowserRefInputSchema, browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema,
},
async ({ ref, browserId }) => {
@@ -581,12 +573,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: toolConfig.command,
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
ref,
},
},
@@ -604,7 +595,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
inputSchema: {
ref: BrowserRefInputSchema,
checked: z.boolean().default(true),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -614,12 +605,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "check",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
ref,
checked,
},
@@ -637,7 +627,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
inputSchema: {
ref: BrowserRefInputSchema,
value: z.string(),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -647,12 +637,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "select",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
ref,
value,
},
@@ -671,7 +660,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
inputSchema: {
sourceRef: BrowserRefInputSchema,
targetRef: BrowserRefInputSchema,
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -681,12 +670,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "drag",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
sourceRef,
targetRef,
},
@@ -704,7 +692,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"Read recent console messages and browser performance network entries for a Paseo desktop browser tab.",
inputSchema: {
maxEntries: z.number().int().positive().max(200).optional(),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -714,12 +702,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "logs",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
maxEntries: maxEntries ?? 50,
},
},
@@ -734,7 +721,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
title: "Read browser storage",
description:
"Read cookies plus localStorage and sessionStorage for a Paseo desktop browser tab.",
inputSchema: { browserId: z.string().min(1).optional() },
inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema,
},
async ({ browserId }) => {
@@ -743,12 +730,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "storage",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
},
},
});
@@ -776,7 +762,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
accuracy: z.number().positive().optional(),
})
.optional(),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -786,12 +772,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "environment",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
...(viewport ? { viewport } : {}),
...(geolocation ? { geolocation } : {}),
},
@@ -806,10 +791,10 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{
title: "Set browser background",
description:
"Set the current page background color in the active Paseo desktop browser tab. Omit browserId to use the active workspace tab; pass a returned browserId only when targeting a specific tab.",
"Set the current page background color in a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.",
inputSchema: {
color: z.string().min(1),
browserId: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema,
},
outputSchema: BrowserToolOutputSchema,
},
@@ -819,12 +804,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
agentId: context.agentId,
cwd: context.cwd,
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
command: {
command: "set_background",
args: {
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
...(browserId ? { browserId } : {}),
browserId,
color,
},
},
@@ -941,20 +925,20 @@ function summarizeBrowserSuccess(
if (payload.result.command === "list_tabs") {
const count = payload.result.tabs.length;
if (count === 0) {
return "No Paseo browser tabs are open. Call browser_new_tab to create one, then use the returned browserId or omit browserId for the active tab.";
return "No Paseo browser tabs are open. Call browser_new_tab to create one.";
}
const tabLines = payload.result.tabs.map((tab) => {
const active = tab.isActive ? " active" : "";
return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url}`;
});
return [
`Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values exactly; do not use 'default'. You may omit browserId to use the active tab.`,
`Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values for tab-scoped browser tools.`,
...tabLines,
].join("\n");
}
if (payload.result.command === "new_tab") {
return `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId exactly, or omit browserId to use this active tab.`;
return `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`;
}
if (payload.result.command === "snapshot") {
@@ -1112,8 +1096,6 @@ function summarizeBrowserError(
return "Browser tools are disabled. Enable desktop browser tools on the host, then try again.";
case "browser_no_desktop":
return "No desktop browser automation client is connected. Open the Paseo desktop app and try again.";
case "browser_no_tab":
return "No active browser tab is available. Call browser_new_tab to create one, then omit browserId for the active tab or use the returned browserId.";
case "browser_timeout":
return "The browser did not respond before the timeout. Try again or check the desktop app.";
case "screenshot_no_frame":
packages/server/src/server/websocket-server.browser-tools.test.ts
+4 -3
View File
@@ -51,6 +51,7 @@ interface QueuedBrowserRequests {
}
const harnesses: BrowserToolsDaemonHarness[] = [];
const BROWSER_ID = "11111111-1111-4111-8111-111111111111";
afterEach(async () => {
await Promise.all(harnesses.splice(0).map((harness) => harness.stop()));
@@ -131,7 +132,7 @@ describe("WebSocketServer browser tools wiring", () => {
});
const resultPromise = harness.broker.execute({
command: { command: "click", args: { ref: "@e1" } },
command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } },
});
const request = await resumedDesktop.nextBrowserRequest();
resumedDesktop.respondToBrowserRequest({
@@ -139,13 +140,13 @@ describe("WebSocketServer browser tools wiring", () => {
payload: {
requestId: request.requestId,
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e1" },
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
},
});
await expect(resultPromise).resolves.toMatchObject({
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e1" },
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
});
});
});