Browser tools: full accessibility snapshot, trusted input, dialogs, evaluate, and tab controls (#1881)
* feat(browser): replace flat snapshot with ARIA tree and page-side refs * feat(browser): trusted CDP input with actionability waits * feat(browser): auto-handle javascript dialogs * feat(browser): add browser_evaluate * feat(browser): add scroll, resize, and close_tab tools * fix(browser): review fixes for snapshot/input/dialog seams * fix(browser): address review findings on dialog capture, truncation, and keypress targets * fix(harness): drop resize check that cannot run under parked compositor surfaces * fix(browser): second review round — snapshot fidelity, key input, dialog scoping
This commit is contained in:
Mohamed Boudra
@@ -221,7 +221,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -234,7 +237,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
});
|
||||
@@ -299,7 +305,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -423,7 +432,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
title: "One",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
secondHost.resolveLatestWith(broker, {
|
||||
@@ -435,7 +447,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://two.example",
|
||||
title: "Two",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -449,6 +464,127 @@ describe("BrowserToolsBroker", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test.each([
|
||||
{
|
||||
name: "scroll",
|
||||
command: { command: "scroll", args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 } },
|
||||
result: { command: "scroll", browserId: BROWSER_ID, deltaX: 0, deltaY: 400 },
|
||||
},
|
||||
{
|
||||
name: "resize",
|
||||
command: { command: "resize", args: { browserId: BROWSER_ID, width: 1024, height: 768 } },
|
||||
result: { command: "resize", browserId: BROWSER_ID, width: 1024, height: 768 },
|
||||
},
|
||||
{
|
||||
name: "close_tab",
|
||||
command: { command: "close_tab", args: { browserId: BROWSER_ID } },
|
||||
result: { command: "close_tab", browserId: BROWSER_ID },
|
||||
},
|
||||
] satisfies Array<{
|
||||
name: string;
|
||||
command: BrowserAutomationCommand;
|
||||
result: BrowserAutomationExecuteResponse["payload"] extends infer Payload
|
||||
? Payload extends { ok: true }
|
||||
? Payload["result"]
|
||||
: never
|
||||
: never;
|
||||
}>)("routes $name to the host that owns the browser id", async ({ command, result }) => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const other = new FakeBrowserHostClient("host-1");
|
||||
const owner = new FakeBrowserHostClient("host-2");
|
||||
broker.registerClient(other);
|
||||
broker.registerClient(owner);
|
||||
|
||||
const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } });
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
},
|
||||
});
|
||||
await newTabPromise;
|
||||
|
||||
const resultPromise = broker.execute({ command, requestId: "req-command" });
|
||||
|
||||
expect(owner.receivedRequests.at(-1)).toEqual({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-command",
|
||||
command,
|
||||
});
|
||||
expect(other.receivedRequests).toEqual([]);
|
||||
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-command",
|
||||
ok: true,
|
||||
result,
|
||||
});
|
||||
|
||||
await expect(resultPromise).resolves.toEqual({
|
||||
requestId: "req-command",
|
||||
ok: true,
|
||||
result,
|
||||
});
|
||||
});
|
||||
|
||||
test("successful close_tab clears browser id host affinity", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const other = new FakeBrowserHostClient("host-1");
|
||||
const owner = new FakeBrowserHostClient("host-2");
|
||||
broker.registerClient(other);
|
||||
broker.registerClient(owner);
|
||||
|
||||
const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } });
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
},
|
||||
});
|
||||
await newTabPromise;
|
||||
|
||||
const closePromise = broker.execute({
|
||||
command: { command: "close_tab", args: { browserId: BROWSER_ID } },
|
||||
requestId: "req-close",
|
||||
});
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-close",
|
||||
ok: true,
|
||||
result: { command: "close_tab", browserId: BROWSER_ID },
|
||||
});
|
||||
await expect(closePromise).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "close_tab", browserId: BROWSER_ID },
|
||||
});
|
||||
|
||||
await expect(
|
||||
broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
requestId: "req-after-close",
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
requestId: "req-after-close",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_tab_not_found",
|
||||
message: `Browser tab ${BROWSER_ID} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`,
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(owner.receivedRequests.at(-1)?.command).toEqual({
|
||||
command: "close_tab",
|
||||
args: { browserId: BROWSER_ID },
|
||||
});
|
||||
expect(other.receivedRequests).toEqual([]);
|
||||
});
|
||||
|
||||
test("failed list tabs aggregation does not seed browser id affinity", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const firstHost = new FakeBrowserHostClient("host-1");
|
||||
@@ -531,6 +667,35 @@ describe("BrowserToolsBroker", () => {
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
broker.execute({
|
||||
command: {
|
||||
command: "evaluate",
|
||||
args: { browserId: BROWSER_ID, function: "() => document.title" },
|
||||
},
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: 'Browser automation command "evaluate" is not supported by the desktop app.',
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
broker.execute({
|
||||
command: { command: "scroll", args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 } },
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: 'Browser automation command "scroll" is not supported by the desktop app.',
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(client.receivedRequests).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -623,7 +788,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -401,6 +401,12 @@ export class BrowserToolsBroker {
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.result.command === "close_tab") {
|
||||
this.browserHostByBrowserId.delete(payload.result.browserId);
|
||||
this.strandedBrowserHostByBrowserId.delete(payload.result.browserId);
|
||||
return;
|
||||
}
|
||||
|
||||
if ("browserId" in payload.result) {
|
||||
this.browserHostByBrowserId.set(payload.result.browserId, clientId);
|
||||
this.strandedBrowserHostByBrowserId.delete(payload.result.browserId);
|
||||
@@ -462,28 +468,10 @@ export class BrowserToolsBroker {
|
||||
}
|
||||
|
||||
function getBrowserIdForCommand(command: BrowserAutomationCommand): string | null {
|
||||
switch (command.command) {
|
||||
case "list_tabs":
|
||||
case "new_tab":
|
||||
return null;
|
||||
case "snapshot":
|
||||
case "click":
|
||||
case "fill":
|
||||
case "wait":
|
||||
case "type":
|
||||
case "keypress":
|
||||
case "navigate":
|
||||
case "back":
|
||||
case "forward":
|
||||
case "reload":
|
||||
case "screenshot":
|
||||
case "upload":
|
||||
case "select":
|
||||
case "hover":
|
||||
case "drag":
|
||||
case "logs":
|
||||
return command.args.browserId;
|
||||
if (command.command === "list_tabs" || command.command === "new_tab") {
|
||||
return null;
|
||||
}
|
||||
return command.args.browserId;
|
||||
}
|
||||
|
||||
function describeBrowserHost(host: RegisteredBrowserHost): string {
|
||||
|
||||
@@ -133,7 +133,10 @@ function snapshotPayload(): Extract<BrowserToolsResponsePayload, { ok: true }> {
|
||||
workspaceId: "wks_workspace_a",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: '- document "Example"\n - button "Save" [ref=@e1]',
|
||||
truncated: false,
|
||||
stats: { nodeCount: 2, refCount: 1, textLength: 50 },
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -158,7 +161,43 @@ const routedToolCases = [
|
||||
name: "click",
|
||||
toolName: "browser_click",
|
||||
input: { browserId: BROWSER_ID, ref: "@e2" },
|
||||
command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e2" } },
|
||||
command: {
|
||||
command: "click",
|
||||
args: {
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e2",
|
||||
button: "left",
|
||||
doubleClick: false,
|
||||
modifiers: [],
|
||||
},
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-click",
|
||||
ok: true,
|
||||
result: { command: "click", browserId: BROWSER_ID, ref: "@e2" },
|
||||
},
|
||||
content: [{ type: "text", text: "Clicked browser element @e2." }],
|
||||
},
|
||||
{
|
||||
name: "click options",
|
||||
toolName: "browser_click",
|
||||
input: {
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e2",
|
||||
button: "right",
|
||||
doubleClick: true,
|
||||
modifiers: ["Control", "Shift"],
|
||||
},
|
||||
command: {
|
||||
command: "click",
|
||||
args: {
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e2",
|
||||
button: "right",
|
||||
doubleClick: true,
|
||||
modifiers: ["Control", "Shift"],
|
||||
},
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-click",
|
||||
ok: true,
|
||||
@@ -344,6 +383,85 @@ const routedToolCases = [
|
||||
},
|
||||
content: [{ type: "text", text: "Dragged browser element @e4 to @e5." }],
|
||||
},
|
||||
{
|
||||
name: "evaluate",
|
||||
toolName: "browser_evaluate",
|
||||
input: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" },
|
||||
command: {
|
||||
command: "evaluate",
|
||||
args: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" },
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-evaluate",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "evaluate",
|
||||
browserId: BROWSER_ID,
|
||||
resultJson: '"Save"',
|
||||
truncated: false,
|
||||
},
|
||||
},
|
||||
content: [{ type: "text", text: 'Browser evaluate returned:\n"Save"' }],
|
||||
},
|
||||
{
|
||||
name: "scroll",
|
||||
toolName: "browser_scroll",
|
||||
input: { browserId: BROWSER_ID, ref: "@e1", deltaX: 0, deltaY: 400 },
|
||||
command: {
|
||||
command: "scroll",
|
||||
args: { browserId: BROWSER_ID, ref: "@e1", deltaX: 0, deltaY: 400 },
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-scroll",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "scroll",
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e1",
|
||||
deltaX: 0,
|
||||
deltaY: 400,
|
||||
},
|
||||
},
|
||||
content: [{ type: "text", text: "Scrolled browser element @e1 by 0, 400." }],
|
||||
},
|
||||
{
|
||||
name: "resize",
|
||||
toolName: "browser_resize",
|
||||
input: { browserId: BROWSER_ID, width: 1024, height: 768 },
|
||||
command: {
|
||||
command: "resize",
|
||||
args: { browserId: BROWSER_ID, width: 1024, height: 768 },
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-resize",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "resize",
|
||||
browserId: BROWSER_ID,
|
||||
width: 1024,
|
||||
height: 768,
|
||||
},
|
||||
},
|
||||
content: [{ type: "text", text: "Resized browser viewport to 1024x768." }],
|
||||
},
|
||||
{
|
||||
name: "close_tab",
|
||||
toolName: "browser_close_tab",
|
||||
input: { browserId: BROWSER_ID },
|
||||
command: {
|
||||
command: "close_tab",
|
||||
args: { browserId: BROWSER_ID },
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-close-tab",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "close_tab",
|
||||
browserId: BROWSER_ID,
|
||||
},
|
||||
},
|
||||
content: [{ type: "text", text: `Closed browser tab ${BROWSER_ID}.` }],
|
||||
},
|
||||
] satisfies Array<{
|
||||
name: string;
|
||||
toolName: string;
|
||||
@@ -459,6 +577,10 @@ describe("registerBrowserTools", () => {
|
||||
"browser_select",
|
||||
"browser_drag",
|
||||
"browser_logs",
|
||||
"browser_evaluate",
|
||||
"browser_scroll",
|
||||
"browser_resize",
|
||||
"browser_close_tab",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -668,7 +790,10 @@ describe("registerBrowserTools", () => {
|
||||
workspaceId: "wks_workspace_a",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: '- document "Example"\n - button "Save" [ref=@e1]',
|
||||
truncated: false,
|
||||
stats: { nodeCount: 2, refCount: 1, textLength: 50 },
|
||||
},
|
||||
context: {
|
||||
agentId: "agent-1",
|
||||
@@ -726,6 +851,89 @@ describe("registerBrowserTools", () => {
|
||||
},
|
||||
);
|
||||
|
||||
test("success responses include handled dialog metadata and a text note", async () => {
|
||||
const harness = new BrowserToolHarness();
|
||||
const payload: Extract<BrowserToolsResponsePayload, { ok: true }> = {
|
||||
requestId: "req-click",
|
||||
ok: true,
|
||||
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
|
||||
dialogs: [
|
||||
{
|
||||
type: "confirm",
|
||||
message: "Delete item?",
|
||||
action: "dismissed",
|
||||
timestamp: 123,
|
||||
},
|
||||
],
|
||||
};
|
||||
harness.broker.setResponse(payload);
|
||||
|
||||
const response = await harness.execute("browser_click", { browserId: BROWSER_ID, ref: "@e1" });
|
||||
|
||||
expect(response.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: 'Clicked browser element @e1.\nHandled browser dialog: dismissed confirm "Delete item?".',
|
||||
},
|
||||
]);
|
||||
expect(response.structuredContent).toEqual({
|
||||
ok: true,
|
||||
result: payload.result,
|
||||
dialogs: payload.dialogs,
|
||||
context: {
|
||||
agentId: "agent-1",
|
||||
cwd: "/repo",
|
||||
workspaceId: "wks_workspace_a",
|
||||
browserId: BROWSER_ID,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("failure responses include handled dialog metadata and a text note", async () => {
|
||||
const harness = new BrowserToolHarness();
|
||||
const payload: Extract<BrowserToolsResponsePayload, { ok: false }> = {
|
||||
requestId: "req-wait",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_timeout",
|
||||
message: "Timed out waiting for browser URL: /next",
|
||||
retryable: true,
|
||||
},
|
||||
dialogs: [
|
||||
{
|
||||
type: "beforeunload",
|
||||
message: "Leave site?",
|
||||
action: "dismissed",
|
||||
timestamp: 124,
|
||||
},
|
||||
],
|
||||
};
|
||||
harness.broker.setResponse(payload);
|
||||
|
||||
const response = await harness.execute("browser_wait", {
|
||||
browserId: BROWSER_ID,
|
||||
url: "/next",
|
||||
});
|
||||
|
||||
expect(response.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: 'The browser did not respond before the timeout. Try again or check the browser host.\nHandled browser dialog: dismissed beforeunload "Leave site?".',
|
||||
},
|
||||
]);
|
||||
expect(response.structuredContent).toEqual({
|
||||
ok: false,
|
||||
error: payload.error,
|
||||
dialogs: payload.dialogs,
|
||||
context: {
|
||||
agentId: "agent-1",
|
||||
cwd: "/repo",
|
||||
workspaceId: "wks_workspace_a",
|
||||
browserId: BROWSER_ID,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("wait rejects calls without a condition", () => {
|
||||
const harness = new BrowserToolHarness();
|
||||
|
||||
|
||||
@@ -45,6 +45,18 @@ const BrowserToolOutputSchema = {
|
||||
retryable: z.boolean(),
|
||||
})
|
||||
.optional(),
|
||||
dialogs: z
|
||||
.array(
|
||||
z.object({
|
||||
type: z.enum(["alert", "confirm", "prompt", "beforeunload"]),
|
||||
message: z.string(),
|
||||
defaultValue: z.string().optional(),
|
||||
action: z.enum(["accepted", "dismissed"]),
|
||||
promptText: z.string().optional(),
|
||||
timestamp: z.number(),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
context: z
|
||||
.object({
|
||||
agentId: z.string().optional(),
|
||||
@@ -70,6 +82,8 @@ const BrowserHttpUrlInputSchema = z
|
||||
return normalized;
|
||||
});
|
||||
const BrowserRefInputSchema = z.string().regex(/^@e\d+$/);
|
||||
const BrowserClickButtonInputSchema = z.enum(["left", "right", "middle"]);
|
||||
const BrowserClickModifierInputSchema = z.enum(["Alt", "Control", "Meta", "Shift"]);
|
||||
const BrowserWaitInputSchema = z
|
||||
.object({
|
||||
text: z.string().min(1).optional(),
|
||||
@@ -178,10 +192,13 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
inputSchema: {
|
||||
ref: BrowserRefInputSchema,
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
button: BrowserClickButtonInputSchema.optional(),
|
||||
doubleClick: z.boolean().optional(),
|
||||
modifiers: z.array(BrowserClickModifierInputSchema).optional(),
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ ref, browserId }) => {
|
||||
async ({ ref, browserId, button, doubleClick, modifiers }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
@@ -193,6 +210,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
args: {
|
||||
browserId,
|
||||
ref,
|
||||
button: button ?? "left",
|
||||
doubleClick: doubleClick ?? false,
|
||||
modifiers: modifiers ?? [],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -605,6 +625,136 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
|
||||
options.registerTool(
|
||||
"browser_evaluate",
|
||||
{
|
||||
title: "Evaluate browser JavaScript",
|
||||
description:
|
||||
"Evaluate a JavaScript function in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; when ref is provided, refs come from the latest browser_snapshot and the resolved element is passed as the first argument.",
|
||||
inputSchema: {
|
||||
function: z.string().min(1),
|
||||
ref: BrowserRefInputSchema.optional(),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ function: functionSource, ref, browserId }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
cwd: context.cwd,
|
||||
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
||||
|
||||
command: {
|
||||
command: "evaluate",
|
||||
args: {
|
||||
browserId,
|
||||
function: functionSource,
|
||||
...(ref ? { ref } : {}),
|
||||
},
|
||||
},
|
||||
});
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
|
||||
options.registerTool(
|
||||
"browser_scroll",
|
||||
{
|
||||
title: "Scroll browser",
|
||||
description:
|
||||
"Scroll a Paseo browser tab by deltaX/deltaY CSS pixels. Use browserId from browser_new_tab or browser_list_tabs; optional ref comes from the latest browser_snapshot and centers the wheel input over that element.",
|
||||
inputSchema: {
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
ref: BrowserRefInputSchema.optional(),
|
||||
deltaX: z.number(),
|
||||
deltaY: z.number(),
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ browserId, ref, deltaX, deltaY }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
cwd: context.cwd,
|
||||
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
||||
|
||||
command: {
|
||||
command: "scroll",
|
||||
args: {
|
||||
browserId,
|
||||
...(ref ? { ref } : {}),
|
||||
deltaX,
|
||||
deltaY,
|
||||
},
|
||||
},
|
||||
});
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
|
||||
options.registerTool(
|
||||
"browser_resize",
|
||||
{
|
||||
title: "Resize browser viewport",
|
||||
description:
|
||||
"Resize a Paseo browser tab's resident webview viewport. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
inputSchema: {
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ browserId, width, height }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
cwd: context.cwd,
|
||||
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
||||
|
||||
command: {
|
||||
command: "resize",
|
||||
args: {
|
||||
browserId,
|
||||
width,
|
||||
height,
|
||||
},
|
||||
},
|
||||
});
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
|
||||
options.registerTool(
|
||||
"browser_close_tab",
|
||||
{
|
||||
title: "Close browser tab",
|
||||
description:
|
||||
"Close a Paseo browser tab, remove its resident webview, and unregister it from the browser automation host. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
inputSchema: {
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ browserId }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
cwd: context.cwd,
|
||||
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
||||
|
||||
command: {
|
||||
command: "close_tab",
|
||||
args: {
|
||||
browserId,
|
||||
},
|
||||
},
|
||||
});
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function resolveBrowserToolContext(options: RegisterBrowserToolsOptions): {
|
||||
@@ -681,16 +831,23 @@ function browserToolResult(params: {
|
||||
structuredContent: {
|
||||
ok: true,
|
||||
result: browserToolStructuredResult(payload.result),
|
||||
...(payload.dialogs ? { dialogs: payload.dialogs } : {}),
|
||||
context,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: "text", text: summarizeBrowserError(payload.error) }],
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: appendDialogSummary(summarizeBrowserError(payload.error), payload.dialogs),
|
||||
},
|
||||
],
|
||||
structuredContent: {
|
||||
ok: false,
|
||||
error: payload.error,
|
||||
...(payload.dialogs ? { dialogs: payload.dialogs } : {}),
|
||||
context,
|
||||
},
|
||||
};
|
||||
@@ -732,34 +889,35 @@ function browserToolImageContent(
|
||||
function summarizeBrowserSuccess(
|
||||
payload: Extract<BrowserToolsResponsePayload, { ok: true }>,
|
||||
): string {
|
||||
const withDialogs = (summary: string) => appendDialogSummary(summary, payload.dialogs);
|
||||
const controlSummary = summarizeBrowserControlSuccess(payload.result);
|
||||
if (controlSummary) {
|
||||
return controlSummary;
|
||||
return withDialogs(controlSummary);
|
||||
}
|
||||
|
||||
const refActionSummary = summarizeBrowserRefActionSuccess(payload.result);
|
||||
if (refActionSummary) {
|
||||
return refActionSummary;
|
||||
return withDialogs(refActionSummary);
|
||||
}
|
||||
|
||||
const diagnosticsSummary = summarizeBrowserDiagnosticsSuccess(payload.result);
|
||||
if (diagnosticsSummary) {
|
||||
return diagnosticsSummary;
|
||||
return withDialogs(diagnosticsSummary);
|
||||
}
|
||||
|
||||
const keyboardSummary = summarizeBrowserKeyboardSuccess(payload.result);
|
||||
if (keyboardSummary) {
|
||||
return keyboardSummary;
|
||||
return withDialogs(keyboardSummary);
|
||||
}
|
||||
|
||||
const navigationSummary = summarizeBrowserNavigationSuccess(payload.result);
|
||||
if (navigationSummary) {
|
||||
return navigationSummary;
|
||||
return withDialogs(navigationSummary);
|
||||
}
|
||||
|
||||
const mediaSummary = summarizeBrowserMediaSuccess(payload.result);
|
||||
if (mediaSummary) {
|
||||
return mediaSummary;
|
||||
return withDialogs(mediaSummary);
|
||||
}
|
||||
|
||||
if (payload.result.command === "list_tabs") {
|
||||
@@ -771,26 +929,49 @@ function summarizeBrowserSuccess(
|
||||
const active = tab.isActive ? " active" : "";
|
||||
return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url}`;
|
||||
});
|
||||
return [
|
||||
`Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values for tab-scoped browser tools.`,
|
||||
...tabLines,
|
||||
].join("\n");
|
||||
return withDialogs(
|
||||
[
|
||||
`Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values for tab-scoped browser tools.`,
|
||||
...tabLines,
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
if (payload.result.command === "new_tab") {
|
||||
return `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`;
|
||||
return withDialogs(
|
||||
`Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (payload.result.command === "snapshot") {
|
||||
const count = payload.result.elements.length;
|
||||
return `Snapshot captured ${count} element${count === 1 ? "" : "s"}.`;
|
||||
return withDialogs(
|
||||
[
|
||||
`Snapshot captured ${payload.result.stats.nodeCount} node${payload.result.stats.nodeCount === 1 ? "" : "s"} with ${payload.result.stats.refCount} ref${payload.result.stats.refCount === 1 ? "" : "s"}.`,
|
||||
`Title: ${payload.result.title || "Untitled"}`,
|
||||
`URL: ${payload.result.url}`,
|
||||
"",
|
||||
payload.result.snapshot,
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
if (payload.result.command === "wait") {
|
||||
return `Browser wait matched ${payload.result.matched}.`;
|
||||
return withDialogs(`Browser wait matched ${payload.result.matched}.`);
|
||||
}
|
||||
|
||||
return `Browser ${payload.result.command} complete.`;
|
||||
return withDialogs(`Browser ${payload.result.command} complete.`);
|
||||
}
|
||||
|
||||
function appendDialogSummary(
|
||||
summary: string,
|
||||
dialogs: BrowserToolsResponsePayload["dialogs"],
|
||||
): string {
|
||||
if (!dialogs || dialogs.length === 0) {
|
||||
return summary;
|
||||
}
|
||||
return `${summary}\nHandled browser dialog${dialogs.length === 1 ? "" : "s"}: ${dialogs
|
||||
.map((dialog) => `${dialog.action} ${dialog.type} ${JSON.stringify(dialog.message)}`)
|
||||
.join("; ")}.`;
|
||||
}
|
||||
|
||||
function summarizeBrowserMediaSuccess(
|
||||
@@ -841,9 +1022,18 @@ function summarizeBrowserNavigationSuccess(
|
||||
function summarizeBrowserDiagnosticsSuccess(
|
||||
result: Extract<BrowserToolsResponsePayload, { ok: true }>["result"],
|
||||
): string | null {
|
||||
if (result.command === "evaluate") {
|
||||
return [
|
||||
"Browser evaluate returned:",
|
||||
result.resultJson,
|
||||
...(result.truncated ? ["Result was truncated."] : []),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (result.command !== "logs") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const consoleCount = result.console.length;
|
||||
const networkCount = result.network.length;
|
||||
return `Read ${consoleCount} console log${consoleCount === 1 ? "" : "s"} and ${networkCount} network entr${networkCount === 1 ? "y" : "ies"}.`;
|
||||
@@ -878,6 +1068,20 @@ function summarizeBrowserControlSuccess(
|
||||
return `Dragged browser element ${result.sourceRef} to ${result.targetRef}.`;
|
||||
}
|
||||
|
||||
if (result.command === "scroll") {
|
||||
return result.ref
|
||||
? `Scrolled browser element ${result.ref} by ${result.deltaX}, ${result.deltaY}.`
|
||||
: `Scrolled browser by ${result.deltaX}, ${result.deltaY}.`;
|
||||
}
|
||||
|
||||
if (result.command === "resize") {
|
||||
return `Resized browser viewport to ${result.width}x${result.height}.`;
|
||||
}
|
||||
|
||||
if (result.command === "close_tab") {
|
||||
return `Closed browser tab ${result.browserId}.`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user