Tighten browser tool schemas

This commit is contained in:
Mohamed Boudra
2026-07-02 20:00:17 +02:00
parent 71ce96b60f
commit 554e525186
2 changed files with 455 additions and 99 deletions
@@ -13,6 +13,9 @@ const BROWSER_ID = "11111111-1111-4111-8111-111111111111";
const BROWSER_ID_MESSAGE = const BROWSER_ID_MESSAGE =
"browserId must be a real id returned by browser_new_tab or browser_list_tabs"; "browserId must be a real id returned by browser_new_tab or browser_list_tabs";
const WAIT_CONDITION_MESSAGE = "browser_wait requires exactly one of text or url"; const WAIT_CONDITION_MESSAGE = "browser_wait requires exactly one of text or url";
const HTTP_URL_MESSAGE = "URL must use http/https only";
const WORKSPACE_CONTEXT_MESSAGE =
"This browser tool needs a workspace. Start the agent from a Paseo workspace before calling browser_new_tab or browser_list_tabs.";
interface RegisteredTool { interface RegisteredTool {
config: PaseoToolConfig; config: PaseoToolConfig;
@@ -65,6 +68,10 @@ class BrowserToolHarness {
return this.get(name).handler(parsed, {}); return this.get(name).handler(parsed, {});
} }
public outputParses(name: string, output: unknown) {
return schemaFor(this.get(name).config.outputSchema).safeParse(output);
}
private get(name: string): RegisteredTool { private get(name: string): RegisteredTool {
const tool = this.tools.get(name); const tool = this.tools.get(name);
if (!tool) { if (!tool) {
@@ -103,6 +110,49 @@ function listTabsPayload(): Extract<BrowserToolsResponsePayload, { ok: true }> {
}; };
} }
function newTabPayload(): Extract<BrowserToolsResponsePayload, { ok: true }> {
return {
requestId: "req-new-tab",
ok: true,
result: {
command: "new_tab",
browserId: BROWSER_ID,
workspaceId: "wks_workspace_a",
url: "https://example.com",
},
};
}
function snapshotPayload(): Extract<BrowserToolsResponsePayload, { ok: true }> {
return {
requestId: "req-snapshot",
ok: true,
result: {
command: "snapshot",
browserId: BROWSER_ID,
workspaceId: "wks_workspace_a",
url: "https://example.com",
title: "Example",
elements: [],
},
};
}
function screenshotPayload(): Extract<BrowserToolsResponsePayload, { ok: true }> {
return {
requestId: "req-screenshot",
ok: true,
result: {
command: "screenshot",
browserId: BROWSER_ID,
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 800,
height: 600,
},
};
}
describe("registerBrowserTools", () => { describe("registerBrowserTools", () => {
test("list tabs sends workspace in the request envelope", async () => { test("list tabs sends workspace in the request envelope", async () => {
const harness = new BrowserToolHarness(); const harness = new BrowserToolHarness();
@@ -127,16 +177,7 @@ describe("registerBrowserTools", () => {
test("new tab sends workspace in the request envelope", async () => { test("new tab sends workspace in the request envelope", async () => {
const harness = new BrowserToolHarness(); const harness = new BrowserToolHarness();
harness.broker.setResponse({ harness.broker.setResponse(newTabPayload());
requestId: "req-new-tab",
ok: true,
result: {
command: "new_tab",
browserId: BROWSER_ID,
workspaceId: "wks_workspace_a",
url: "https://example.com",
},
});
const response = await harness.execute("browser_new_tab", { url: "https://example.com" }); const response = await harness.execute("browser_new_tab", { url: "https://example.com" });
@@ -156,6 +197,155 @@ describe("registerBrowserTools", () => {
]); ]);
}); });
test.each([
{
name: "navigate accepts localhost without a scheme as http",
toolName: "browser_navigate",
input: { browserId: BROWSER_ID, url: "localhost:3000" },
expected: { browserId: BROWSER_ID, url: "http://localhost:3000" },
},
{
name: "download accepts an IP host without a scheme as http",
toolName: "browser_download",
input: { browserId: BROWSER_ID, url: "127.0.0.1:8081/admin" },
expected: { browserId: BROWSER_ID, url: "http://127.0.0.1:8081/admin" },
},
{
name: "new tab accepts a domain path without a scheme as http",
toolName: "browser_new_tab",
input: { url: "example.com/x" },
expected: { url: "http://example.com/x" },
},
{
name: "navigate keeps https URLs unchanged",
toolName: "browser_navigate",
input: { browserId: BROWSER_ID, url: "https://example.com/x" },
expected: { browserId: BROWSER_ID, url: "https://example.com/x" },
},
])("$name", ({ toolName, input, expected }) => {
const harness = new BrowserToolHarness();
const parsed = harness.validate(toolName, input);
expect(parsed).toEqual({ success: true, data: expected });
});
test.each([
{
name: "navigate rejects file URLs",
toolName: "browser_navigate",
input: { browserId: BROWSER_ID, url: "file:///tmp/index.html" },
},
{
name: "download rejects file URLs",
toolName: "browser_download",
input: { browserId: BROWSER_ID, url: "file:///tmp/archive.zip" },
},
{
name: "new tab rejects file URLs",
toolName: "browser_new_tab",
input: { url: "file:///tmp/index.html" },
},
])("$name", ({ toolName, input }) => {
const harness = new BrowserToolHarness();
const parsed = harness.validate(toolName, input);
expect(parsed).toMatchObject({
success: false,
error: { issues: [expect.objectContaining({ message: HTTP_URL_MESSAGE })] },
});
});
test("list tabs tells agents without a workspace how to proceed", async () => {
const harness = new BrowserToolHarness({ id: "agent-1", cwd: "/repo" });
const response = await harness.execute("browser_list_tabs", {});
expect(harness.broker.calls).toEqual([]);
expect(response.content).toEqual([{ type: "text", text: WORKSPACE_CONTEXT_MESSAGE }]);
expect(response.structuredContent).toEqual({
ok: false,
error: {
code: "browser_denied",
message: WORKSPACE_CONTEXT_MESSAGE,
retryable: false,
},
context: {
agentId: "agent-1",
cwd: "/repo",
},
});
});
test("new tab tells agents without a workspace how to proceed", async () => {
const harness = new BrowserToolHarness({ id: "agent-1", cwd: "/repo" });
const response = await harness.execute("browser_new_tab", {});
expect(harness.broker.calls).toEqual([]);
expect(response.content).toEqual([{ type: "text", text: WORKSPACE_CONTEXT_MESSAGE }]);
expect(response.structuredContent).toEqual({
ok: false,
error: {
code: "browser_denied",
message: WORKSPACE_CONTEXT_MESSAGE,
retryable: false,
},
context: {
agentId: "agent-1",
cwd: "/repo",
},
});
});
test("new tab declares the shape it returns", async () => {
const harness = new BrowserToolHarness();
harness.broker.setResponse(newTabPayload());
const response = await harness.execute("browser_new_tab", {});
expect(harness.outputParses("browser_new_tab", response.structuredContent)).toEqual({
success: true,
data: response.structuredContent,
});
});
test("list tabs declares the shape it returns", async () => {
const harness = new BrowserToolHarness();
const response = await harness.execute("browser_list_tabs", {});
expect(harness.outputParses("browser_list_tabs", response.structuredContent)).toEqual({
success: true,
data: response.structuredContent,
});
});
test("snapshot declares the shape it returns", async () => {
const harness = new BrowserToolHarness();
harness.broker.setResponse(snapshotPayload());
const response = await harness.execute("browser_snapshot", { browserId: BROWSER_ID });
expect(harness.outputParses("browser_snapshot", response.structuredContent)).toEqual({
success: true,
data: response.structuredContent,
});
});
test("screenshot declares the shape it returns", async () => {
const harness = new BrowserToolHarness();
harness.broker.setResponse(screenshotPayload());
const response = await harness.execute("browser_screenshot", { browserId: BROWSER_ID });
expect(harness.outputParses("browser_screenshot", response.structuredContent)).toEqual({
success: true,
data: response.structuredContent,
});
});
test("snapshot rejects calls without a browser id", () => { test("snapshot rejects calls without a browser id", () => {
const harness = new BrowserToolHarness(); const harness = new BrowserToolHarness();
@@ -180,18 +370,7 @@ describe("registerBrowserTools", () => {
test("snapshot sends browser id in command args only", async () => { test("snapshot sends browser id in command args only", async () => {
const harness = new BrowserToolHarness(); const harness = new BrowserToolHarness();
harness.broker.setResponse({ harness.broker.setResponse(snapshotPayload());
requestId: "req-snapshot",
ok: true,
result: {
command: "snapshot",
browserId: BROWSER_ID,
workspaceId: "wks_workspace_a",
url: "https://example.com",
title: "Example",
elements: [],
},
});
const response = await harness.execute("browser_snapshot", { browserId: BROWSER_ID }); const response = await harness.execute("browser_snapshot", { browserId: BROWSER_ID });
@@ -288,16 +467,41 @@ describe("registerBrowserTools", () => {
expect(response.content).toEqual([{ type: "text", text: "Browser wait matched text." }]); expect(response.content).toEqual([{ type: "text", text: "Browser wait matched text." }]);
}); });
test("tools keep empty context when there is no caller agent", async () => { test("tab tools keep empty context when there is no caller agent", async () => {
const harness = new BrowserToolHarness(null, null); const harness = new BrowserToolHarness(null, null);
harness.broker.setResponse({
requestId: "req-page-info",
ok: true,
result: {
command: "page_info",
tab: {
browserId: BROWSER_ID,
url: "https://example.com",
title: "Example",
isActive: false,
isLoading: false,
},
},
});
const response = await harness.execute("browser_list_tabs", {}); const response = await harness.execute("browser_page_info", { browserId: BROWSER_ID });
expect(harness.broker.calls).toEqual([{ command: { command: "list_tabs", args: {} } }]); expect(harness.broker.calls).toEqual([
{ command: { command: "page_info", args: { browserId: BROWSER_ID } } },
]);
expect(response.structuredContent).toEqual({ expect(response.structuredContent).toEqual({
ok: true, ok: true,
result: listTabsPayload().result, result: {
context: {}, command: "page_info",
tab: {
browserId: BROWSER_ID,
url: "https://example.com",
title: "Example",
isActive: false,
isLoading: false,
},
},
context: { browserId: BROWSER_ID },
}); });
}); });
}); });
+224 -72
View File
@@ -1,5 +1,36 @@
import { z } from "zod"; import { z } from "zod";
import { BrowserAutomationBrowserIdSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import {
BrowserAutomationBackResultSchema,
BrowserAutomationBrowserIdSchema,
BrowserAutomationCheckResultSchema,
BrowserAutomationClearResultSchema,
BrowserAutomationClickResultSchema,
BrowserAutomationDownloadResultSchema,
BrowserAutomationDragResultSchema,
BrowserAutomationEnvironmentResultSchema,
BrowserAutomationErrorSchema,
BrowserAutomationFillResultSchema,
BrowserAutomationFocusResultSchema,
BrowserAutomationForwardResultSchema,
BrowserAutomationFullPageScreenshotResultSchema,
BrowserAutomationHoverResultSchema,
BrowserAutomationKeypressResultSchema,
BrowserAutomationListTabsResultSchema,
BrowserAutomationLogsResultSchema,
BrowserAutomationNavigateResultSchema,
BrowserAutomationNewTabResultSchema,
BrowserAutomationPageInfoResultSchema,
BrowserAutomationPdfResultSchema,
BrowserAutomationReloadResultSchema,
BrowserAutomationScreenshotResultSchema,
BrowserAutomationSelectResultSchema,
BrowserAutomationSetBackgroundResultSchema,
BrowserAutomationSnapshotResultSchema,
BrowserAutomationStorageResultSchema,
BrowserAutomationTypeResultSchema,
BrowserAutomationUploadResultSchema,
BrowserAutomationWaitResultSchema,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { BrowserToolsBroker } from "./broker.js"; import type { BrowserToolsBroker } from "./broker.js";
import type { BrowserToolsResponsePayload } from "./errors.js"; import type { BrowserToolsResponsePayload } from "./errors.js";
import type { import type {
@@ -29,26 +60,50 @@ export interface RegisterBrowserToolsOptions {
resolveCallerAgent: () => CallerAgentContext | null; resolveCallerAgent: () => CallerAgentContext | null;
} }
const BrowserToolOutputSchema = z.object({ const HTTP_URL_ONLY_MESSAGE = "URL must use http/https only";
ok: z.boolean(), const WORKSPACE_CONTEXT_MESSAGE =
result: z.unknown().optional(), "This browser tool needs a workspace. Start the agent from a Paseo workspace before calling browser_new_tab or browser_list_tabs.";
error: z const SCHEMELESS_HTTP_URL_PATTERN =
.object({ /^(?:localhost|(?:\d{1,3}\.){3}\d{1,3}|\[[0-9a-f:.]+\]|[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+)(?::\d{1,5})?(?:[/?#].*)?$/i;
code: z.string(),
message: z.string(), const BrowserToolContextSchema = z.object({
retryable: z.boolean(), agentId: z.string().optional(),
}) cwd: z.string().optional(),
.optional(), workspaceId: z.string().optional(),
context: z browserId: z.string().optional(),
.object({
agentId: z.string().optional(),
cwd: z.string().optional(),
workspaceId: z.string().optional(),
browserId: z.string().optional(),
})
.optional(),
}); });
const BrowserToolErrorOutputSchema = z.object({
ok: z.literal(false),
error: BrowserAutomationErrorSchema,
context: BrowserToolContextSchema,
});
function browserToolOutputSchema<ResultSchema extends z.ZodType>(resultSchema: ResultSchema) {
return z.discriminatedUnion("ok", [
z.object({
ok: z.literal(true),
result: resultSchema,
context: BrowserToolContextSchema,
}),
BrowserToolErrorOutputSchema,
]);
}
const BrowserHttpUrlInputSchema = z
.string()
.trim()
.transform((value, context) => {
const normalized = normalizeHttpUrlInput(value);
if (!normalized) {
context.addIssue({
code: "custom",
message: HTTP_URL_ONLY_MESSAGE,
});
return z.NEVER;
}
return normalized;
});
const BrowserRefInputSchema = z.string().regex(/^@e\d+$/); const BrowserRefInputSchema = z.string().regex(/^@e\d+$/);
const BrowserWaitInputSchema = z const BrowserWaitInputSchema = z
.object({ .object({
@@ -61,18 +116,53 @@ const BrowserWaitInputSchema = z
message: "browser_wait requires exactly one of text or url", message: "browser_wait requires exactly one of text or url",
}); });
const BrowserToolOutputs = {
listTabs: browserToolOutputSchema(BrowserAutomationListTabsResultSchema),
newTab: browserToolOutputSchema(BrowserAutomationNewTabResultSchema),
pageInfo: browserToolOutputSchema(BrowserAutomationPageInfoResultSchema),
snapshot: browserToolOutputSchema(BrowserAutomationSnapshotResultSchema),
click: browserToolOutputSchema(BrowserAutomationClickResultSchema),
fill: browserToolOutputSchema(BrowserAutomationFillResultSchema),
wait: browserToolOutputSchema(BrowserAutomationWaitResultSchema),
type: browserToolOutputSchema(BrowserAutomationTypeResultSchema),
keypress: browserToolOutputSchema(BrowserAutomationKeypressResultSchema),
navigate: browserToolOutputSchema(BrowserAutomationNavigateResultSchema),
back: browserToolOutputSchema(BrowserAutomationBackResultSchema),
forward: browserToolOutputSchema(BrowserAutomationForwardResultSchema),
reload: browserToolOutputSchema(BrowserAutomationReloadResultSchema),
screenshot: browserToolOutputSchema(BrowserAutomationScreenshotResultSchema),
fullPageScreenshot: browserToolOutputSchema(BrowserAutomationFullPageScreenshotResultSchema),
pdf: browserToolOutputSchema(BrowserAutomationPdfResultSchema),
download: browserToolOutputSchema(BrowserAutomationDownloadResultSchema),
upload: browserToolOutputSchema(BrowserAutomationUploadResultSchema),
focus: browserToolOutputSchema(BrowserAutomationFocusResultSchema),
clear: browserToolOutputSchema(BrowserAutomationClearResultSchema),
check: browserToolOutputSchema(BrowserAutomationCheckResultSchema),
select: browserToolOutputSchema(BrowserAutomationSelectResultSchema),
hover: browserToolOutputSchema(BrowserAutomationHoverResultSchema),
drag: browserToolOutputSchema(BrowserAutomationDragResultSchema),
logs: browserToolOutputSchema(BrowserAutomationLogsResultSchema),
storage: browserToolOutputSchema(BrowserAutomationStorageResultSchema),
environment: browserToolOutputSchema(BrowserAutomationEnvironmentResultSchema),
setBackground: browserToolOutputSchema(BrowserAutomationSetBackgroundResultSchema),
} as const;
export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { export function registerBrowserTools(options: RegisterBrowserToolsOptions): void {
options.registerTool( options.registerTool(
"browser_list_tabs", "browser_list_tabs",
{ {
title: "List browser tabs", title: "List browser tabs",
description: description:
"List open Paseo desktop browser tabs for this agent's workspace context. Use the returned browserId values for tab-scoped browser tools.", "List open Paseo desktop browser tabs for this agent's workspace. Use returned browserId values with tab-scoped tools.",
inputSchema: {}, inputSchema: {},
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.listTabs,
}, },
async () => { async () => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
const missingWorkspace = requireWorkspaceContext(context);
if (missingWorkspace) {
return missingWorkspace;
}
const payload = await options.broker.execute({ const payload = await options.broker.execute({
agentId: context.agentId, agentId: context.agentId,
cwd: context.cwd, cwd: context.cwd,
@@ -91,14 +181,18 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Create browser tab", title: "Create browser tab",
description: description:
"Create and focus a new Paseo desktop browser tab in this agent's workspace. Optionally pass an http(s) URL to open immediately. Returns the browserId for later tab-scoped browser tools.", "Create and focus a new Paseo desktop browser tab in this agent's workspace. Pass an http(s) URL or a scheme-less host URL, which is treated as http; the returned browserId is used by tab-scoped tools.",
inputSchema: { inputSchema: {
url: z.string().url().optional(), url: BrowserHttpUrlInputSchema.optional(),
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.newTab,
}, },
async ({ url }) => { async ({ url }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
const missingWorkspace = requireWorkspaceContext(context);
if (missingWorkspace) {
return missingWorkspace;
}
const payload = await options.broker.execute({ const payload = await options.broker.execute({
agentId: context.agentId, agentId: context.agentId,
cwd: context.cwd, cwd: context.cwd,
@@ -117,11 +211,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Get browser page info", title: "Get browser page info",
description: description:
"Get page info for a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.", "Get page info for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { inputSchema: {
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.pageInfo,
}, },
async ({ browserId }) => { async ({ browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -146,11 +240,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Snapshot browser page", title: "Snapshot browser page",
description: description:
"Return a model-readable snapshot of a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs. Snapshot refs like @e1 are valid until the page changes or a new snapshot is taken.", "Return a model-readable snapshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: { inputSchema: {
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.snapshot,
}, },
async ({ browserId }) => { async ({ browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -175,12 +269,12 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Click browser element", title: "Click browser element",
description: description:
"Click an element ref from the latest browser_snapshot for this agent's workspace browser tab.", "Click an element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: { inputSchema: {
ref: BrowserRefInputSchema, ref: BrowserRefInputSchema,
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.click,
}, },
async ({ ref, browserId }) => { async ({ ref, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -206,13 +300,13 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Fill browser element", title: "Fill browser element",
description: description:
"Fill an input-like element ref from the latest browser_snapshot for this agent's workspace browser tab.", "Fill an input-like element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: { inputSchema: {
ref: BrowserRefInputSchema, ref: BrowserRefInputSchema,
value: z.string(), value: z.string(),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.fill,
}, },
async ({ ref, value, browserId }) => { async ({ ref, value, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -239,9 +333,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Wait for browser condition", title: "Wait for browser condition",
description: description:
"Wait until a Paseo desktop browser tab contains text or reaches a URL fragment. Pass browserId from browser_new_tab or browser_list_tabs.", "Wait until a Paseo desktop browser tab contains text or reaches a URL fragment. Use browserId from browser_new_tab or browser_list_tabs; waits up to 5s by default on the desktop side.",
inputSchema: BrowserWaitInputSchema, inputSchema: BrowserWaitInputSchema,
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.wait,
}, },
async ({ text, url, timeoutMs, browserId }) => { async ({ text, url, timeoutMs, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -270,13 +364,13 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Type into browser", title: "Type into browser",
description: description:
"Type text into an element ref from the latest browser_snapshot, or into the currently focused browser element when ref is omitted.", "Type text into an element, or into the focused element when ref is omitted. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: { inputSchema: {
text: z.string(), text: z.string(),
ref: BrowserRefInputSchema.optional(), ref: BrowserRefInputSchema.optional(),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.type,
}, },
async ({ text, ref, browserId }) => { async ({ text, ref, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -303,13 +397,13 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Press browser key", title: "Press browser key",
description: description:
"Dispatch a keypress to an element ref from the latest browser_snapshot, or to the currently focused browser element when ref is omitted.", "Dispatch a keypress to an element, or to the focused element when ref is omitted. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: { inputSchema: {
key: z.string().min(1), key: z.string().min(1),
ref: BrowserRefInputSchema.optional(), ref: BrowserRefInputSchema.optional(),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.keypress,
}, },
async ({ key, ref, browserId }) => { async ({ key, ref, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -336,9 +430,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Navigate browser", title: "Navigate browser",
description: description:
"Navigate a Paseo desktop browser tab to a URL. Pass browserId from browser_new_tab or browser_list_tabs.", "Navigate a Paseo desktop browser tab to a URL. Use browserId from browser_new_tab or browser_list_tabs; pass an http(s) URL or a scheme-less host URL, which is treated as http.",
inputSchema: { url: z.string().min(1), browserId: BrowserAutomationBrowserIdSchema }, inputSchema: { url: BrowserHttpUrlInputSchema, browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.navigate,
}, },
async ({ url, browserId }) => { async ({ url, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -365,9 +459,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
name, name,
{ {
title: `Browser ${command}`, title: `Browser ${command}`,
description: `${command} a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.`, description: `${command} a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.`,
inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs[command],
}, },
async ({ browserId }) => { async ({ browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -393,9 +487,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Capture browser screenshot", title: "Capture browser screenshot",
description: description:
"Capture a PNG screenshot of a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.", "Capture a PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.screenshot,
}, },
async ({ browserId }) => { async ({ browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -419,9 +513,10 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"browser_full_page_screenshot", "browser_full_page_screenshot",
{ {
title: "Capture full-page browser screenshot", title: "Capture full-page browser screenshot",
description: "Capture a full-page PNG screenshot of a Paseo desktop browser tab.", description:
"Capture a full-page PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.fullPageScreenshot,
}, },
async ({ browserId }) => { async ({ browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -446,13 +541,13 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Export browser page PDF", title: "Export browser page PDF",
description: description:
"Export a Paseo desktop browser tab as a PDF. Pass browserId from browser_new_tab or browser_list_tabs.", "Export a Paseo desktop browser tab as a PDF. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { inputSchema: {
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
landscape: z.boolean().optional(), landscape: z.boolean().optional(),
printBackground: z.boolean().default(true), printBackground: z.boolean().default(true),
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.pdf,
}, },
async ({ browserId, landscape, printBackground }) => { async ({ browserId, landscape, printBackground }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -478,13 +573,14 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"browser_download", "browser_download",
{ {
title: "Download file in browser", title: "Download file in browser",
description: "Download a URL through the Paseo desktop browser session.", description:
"Download a URL through a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; pass an http(s) URL or a scheme-less host URL, which is treated as http.",
inputSchema: { inputSchema: {
url: z.string().min(1), url: BrowserHttpUrlInputSchema,
fileName: z.string().min(1).optional(), fileName: z.string().min(1).optional(),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.download,
}, },
async ({ url, fileName, browserId }) => { async ({ url, fileName, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -511,13 +607,13 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Upload files in browser", title: "Upload files in browser",
description: description:
"Set workspace files on a file input ref from the latest browser_snapshot. Paths must resolve inside the agent workspace.", "Set workspace files on a file input in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: { inputSchema: {
ref: BrowserRefInputSchema, ref: BrowserRefInputSchema,
filePaths: z.array(z.string().min(1)).min(1), filePaths: z.array(z.string().min(1)).min(1),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.upload,
}, },
async ({ ref, filePaths, browserId }) => { async ({ ref, filePaths, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -544,19 +640,22 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
name: "browser_focus", name: "browser_focus",
command: "focus", command: "focus",
title: "Focus browser element", title: "Focus browser element",
description: "Focus an element ref from the latest browser_snapshot.", description:
"Focus an element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
}, },
{ {
name: "browser_clear", name: "browser_clear",
command: "clear", command: "clear",
title: "Clear browser element", title: "Clear browser element",
description: "Clear an input-like element ref from the latest browser_snapshot.", description:
"Clear an input-like element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
}, },
{ {
name: "browser_hover", name: "browser_hover",
command: "hover", command: "hover",
title: "Hover browser element", title: "Hover browser element",
description: "Hover an element ref from the latest browser_snapshot.", description:
"Hover an element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
}, },
] as const) { ] as const) {
options.registerTool( options.registerTool(
@@ -565,7 +664,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
title: toolConfig.title, title: toolConfig.title,
description: toolConfig.description, description: toolConfig.description,
inputSchema: { ref: BrowserRefInputSchema, browserId: BrowserAutomationBrowserIdSchema }, inputSchema: { ref: BrowserRefInputSchema, browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs[toolConfig.command],
}, },
async ({ ref, browserId }) => { async ({ ref, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -591,13 +690,14 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"browser_check", "browser_check",
{ {
title: "Check browser control", title: "Check browser control",
description: "Set a checkbox or radio ref from the latest browser_snapshot.", description:
"Set a checkbox or radio control in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: { inputSchema: {
ref: BrowserRefInputSchema, ref: BrowserRefInputSchema,
checked: z.boolean().default(true), checked: z.boolean().default(true),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.check,
}, },
async ({ ref, checked, browserId }) => { async ({ ref, checked, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -623,13 +723,14 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"browser_select", "browser_select",
{ {
title: "Select browser option", title: "Select browser option",
description: "Set a select element ref from the latest browser_snapshot to a value.", description:
"Set a select element in a Paseo desktop browser tab to a value. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: { inputSchema: {
ref: BrowserRefInputSchema, ref: BrowserRefInputSchema,
value: z.string(), value: z.string(),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.select,
}, },
async ({ ref, value, browserId }) => { async ({ ref, value, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -656,13 +757,13 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Drag browser element", title: "Drag browser element",
description: description:
"Drag a source element ref onto a target element ref from the latest browser_snapshot.", "Drag one element onto another in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
inputSchema: { inputSchema: {
sourceRef: BrowserRefInputSchema, sourceRef: BrowserRefInputSchema,
targetRef: BrowserRefInputSchema, targetRef: BrowserRefInputSchema,
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.drag,
}, },
async ({ sourceRef, targetRef, browserId }) => { async ({ sourceRef, targetRef, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -689,12 +790,12 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Read browser logs", title: "Read browser logs",
description: description:
"Read recent console messages and browser performance network entries for a Paseo desktop browser tab.", "Read recent console messages and browser performance network entries for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; maxEntries defaults to 50.",
inputSchema: { inputSchema: {
maxEntries: z.number().int().positive().max(200).optional(), maxEntries: z.number().int().positive().max(200).optional(),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.logs,
}, },
async ({ maxEntries, browserId }) => { async ({ maxEntries, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -720,9 +821,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Read browser storage", title: "Read browser storage",
description: description:
"Read cookies plus localStorage and sessionStorage for a Paseo desktop browser tab.", "Read cookies plus localStorage and sessionStorage for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, inputSchema: { browserId: BrowserAutomationBrowserIdSchema },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.storage,
}, },
async ({ browserId }) => { async ({ browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -746,7 +847,8 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
"browser_environment", "browser_environment",
{ {
title: "Set/read browser environment", title: "Set/read browser environment",
description: "Set or read viewport and geolocation for a Paseo desktop browser tab.", description:
"Set or read viewport and geolocation for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { inputSchema: {
viewport: z viewport: z
.object({ .object({
@@ -764,7 +866,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
.optional(), .optional(),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.environment,
}, },
async ({ viewport, geolocation, browserId }) => { async ({ viewport, geolocation, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -791,12 +893,12 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
{ {
title: "Set browser background", title: "Set browser background",
description: description:
"Set the current page background color in a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.", "Set the current page background color in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
inputSchema: { inputSchema: {
color: z.string().min(1), color: z.string().min(1),
browserId: BrowserAutomationBrowserIdSchema, browserId: BrowserAutomationBrowserIdSchema,
}, },
outputSchema: BrowserToolOutputSchema, outputSchema: BrowserToolOutputs.setBackground,
}, },
async ({ color, browserId }) => { async ({ color, browserId }) => {
const context = resolveBrowserToolContext(options); const context = resolveBrowserToolContext(options);
@@ -831,6 +933,56 @@ function resolveBrowserToolContext(options: RegisterBrowserToolsOptions): {
}; };
} }
function normalizeHttpUrlInput(value: string): string | null {
if (value.length === 0) {
return null;
}
const explicitHttpUrl = /^https?:\/\//i.test(value);
if (explicitHttpUrl) {
return isValidHttpUrl(value) ? value : null;
}
if (SCHEMELESS_HTTP_URL_PATTERN.test(value)) {
const normalized = `http://${value}`;
return isValidHttpUrl(normalized) ? normalized : null;
}
return null;
}
function isValidHttpUrl(value: string): boolean {
try {
const url = new URL(value);
return url.protocol === "http:" || url.protocol === "https:";
} catch {
return false;
}
}
function requireWorkspaceContext(context: {
agentId?: string;
cwd?: string;
workspaceId?: string;
}): PaseoToolResult | null {
if (context.workspaceId) {
return null;
}
return browserToolResult({
payload: {
requestId: "browser-tools-workspace-context",
ok: false,
error: {
code: "browser_denied",
message: WORKSPACE_CONTEXT_MESSAGE,
retryable: false,
},
},
context,
});
}
function browserToolResult(params: { function browserToolResult(params: {
payload: BrowserToolsResponsePayload; payload: BrowserToolsResponsePayload;
context: { agentId?: string; cwd?: string; workspaceId?: string; browserId?: string }; context: { agentId?: string; cwd?: string; workspaceId?: string; browserId?: string };