jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Browser tools: full accessibility snapshot, trusted input, dialogs, evaluate, and tab controls (#1881)

Browse Source 
* feat(browser): replace flat snapshot with ARIA tree and page-side refs

* feat(browser): trusted CDP input with actionability waits

* feat(browser): auto-handle javascript dialogs

* feat(browser): add browser_evaluate

* feat(browser): add scroll, resize, and close_tab tools

* fix(browser): review fixes for snapshot/input/dialog seams

* fix(browser): address review findings on dialog capture, truncation, and keypress targets

* fix(harness): drop resize check that cannot run under parked compositor surfaces

* fix(browser): second review round — snapshot fidelity, key input, dialog scoping
This commit is contained in:
Mohamed Boudra
2026-07-04 01:38:07 +02:00
committed by GitHub
parent b28f291493
commit 9c220c2451
29 changed files with 5653 additions and 949 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/protocol/src/browser-automation/rpc-schemas.test.ts
+260 -22
View File
@@ -15,7 +15,39 @@ const commandParseCases = [
{
name: "click",
command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } },
expected: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } },
expected: {
command: "click",
args: {
browserId: BROWSER_ID,
ref: "@e1",
button: "left",
doubleClick: false,
modifiers: [],
},
},
},
{
name: "click options",
command: {
command: "click",
args: {
browserId: BROWSER_ID,
ref: "@e1",
button: "right",
doubleClick: true,
modifiers: ["Meta", "Shift"],
},
},
expected: {
command: "click",
args: {
browserId: BROWSER_ID,
ref: "@e1",
button: "right",
doubleClick: true,
modifiers: ["Meta", "Shift"],
},
},
},
{
name: "fill",
@@ -123,6 +155,72 @@ const commandParseCases = [
command: { command: "logs", args: { browserId: BROWSER_ID } },
expected: { command: "logs", args: { browserId: BROWSER_ID, maxEntries: 50 } },
},
{
name: "evaluate",
command: {
command: "evaluate",
args: { browserId: BROWSER_ID, function: "() => document.title" },
},
expected: {
command: "evaluate",
args: { browserId: BROWSER_ID, function: "() => document.title" },
},
},
{
name: "evaluate with ref",
command: {
command: "evaluate",
args: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" },
},
expected: {
command: "evaluate",
args: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" },
},
},
{
name: "scroll",
command: {
command: "scroll",
args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 },
},
expected: {
command: "scroll",
args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 },
},
},
{
name: "scroll with ref",
command: {
command: "scroll",
args: { browserId: BROWSER_ID, ref: "@e1", deltaX: 10, deltaY: -20 },
},
expected: {
command: "scroll",
args: { browserId: BROWSER_ID, ref: "@e1", deltaX: 10, deltaY: -20 },
},
},
{
name: "resize",
command: {
command: "resize",
args: { browserId: BROWSER_ID, width: 1024, height: 768 },
},
expected: {
command: "resize",
args: { browserId: BROWSER_ID, width: 1024, height: 768 },
},
},
{
name: "close_tab",
command: {
command: "close_tab",
args: { browserId: BROWSER_ID },
},
expected: {
command: "close_tab",
args: { browserId: BROWSER_ID },
},
},
] as const;
const resultParseCases = [
@@ -134,16 +232,10 @@ const resultParseCases = [
workspaceId: "workspace-1",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name", type: "text" },
},
],
format: "aria-yaml",
snapshot: '- document "Fixture"\n - textbox "Name" [ref=@e1]',
truncated: false,
stats: { nodeCount: 2, refCount: 1, textLength: 52, iframeCount: 0, maxDepth: 1 },
},
expected: {
command: "snapshot",
@@ -151,16 +243,10 @@ const resultParseCases = [
workspaceId: "workspace-1",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name", type: "text" },
},
],
format: "aria-yaml",
snapshot: '- document "Fixture"\n - textbox "Name" [ref=@e1]',
truncated: false,
stats: { nodeCount: 2, refCount: 1, textLength: 52, iframeCount: 0, maxDepth: 1 },
},
},
{
@@ -305,6 +391,68 @@ const resultParseCases = [
],
},
},
{
name: "evaluate",
result: {
command: "evaluate",
browserId: BROWSER_ID,
resultJson: '{"title":"Fixture"}',
truncated: false,
},
expected: {
command: "evaluate",
browserId: BROWSER_ID,
resultJson: '{"title":"Fixture"}',
truncated: false,
},
},
{
name: "scroll",
result: {
command: "scroll",
browserId: BROWSER_ID,
ref: "@e1",
deltaX: 10,
deltaY: 400,
x: 40,
y: 30,
},
expected: {
command: "scroll",
browserId: BROWSER_ID,
ref: "@e1",
deltaX: 10,
deltaY: 400,
x: 40,
y: 30,
},
},
{
name: "resize",
result: {
command: "resize",
browserId: BROWSER_ID,
width: 1024,
height: 768,
},
expected: {
command: "resize",
browserId: BROWSER_ID,
width: 1024,
height: 768,
},
},
{
name: "close_tab",
result: {
command: "close_tab",
browserId: BROWSER_ID,
},
expected: {
command: "close_tab",
browserId: BROWSER_ID,
},
},
] as const;
describe("browser automation execute RPC schemas", () => {
@@ -526,7 +674,10 @@ describe("browser automation execute RPC schemas", () => {
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
elements: [],
format: "aria-yaml",
snapshot: "- document",
truncated: false,
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
},
},
});
@@ -557,6 +708,53 @@ describe("browser automation execute RPC schemas", () => {
},
);
test("success responses can report handled dialogs", () => {
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-click",
ok: true,
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
dialogs: [
{
type: "alert",
message: "Saved",
action: "accepted",
timestamp: 123,
},
{
type: "prompt",
message: "Name?",
defaultValue: "Maya",
action: "dismissed",
timestamp: 124,
},
],
},
});
expect(parsed.payload).toEqual({
requestId: "req-click",
ok: true,
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
dialogs: [
{
type: "alert",
message: "Saved",
action: "accepted",
timestamp: 123,
},
{
type: "prompt",
message: "Name?",
defaultValue: "Maya",
action: "dismissed",
timestamp: 124,
},
],
});
});
test("error responses keep stable codes, messages, and retry defaults", () => {
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
@@ -580,4 +778,44 @@ describe("browser automation execute RPC schemas", () => {
},
});
});
test("failure responses can report handled dialogs", () => {
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-navigate",
ok: false,
error: {
code: "browser_timeout",
message: "Timed out waiting for browser URL: /next",
},
dialogs: [
{
type: "beforeunload",
message: "Leave site?",
action: "dismissed",
timestamp: 200,
},
],
},
});
expect(parsed.payload).toEqual({
requestId: "req-navigate",
ok: false,
error: {
code: "browser_timeout",
message: "Timed out waiting for browser URL: /next",
retryable: false,
},
dialogs: [
{
type: "beforeunload",
message: "Leave site?",
action: "dismissed",
timestamp: 200,
},
],
});
});
});
packages/protocol/src/browser-automation/rpc-schemas.ts
+113 -9
View File
@@ -39,6 +39,10 @@ export const BROWSER_AUTOMATION_COMMAND_NAMES = [
"hover",
"drag",
"logs",
"evaluate",
"scroll",
"resize",
"close_tab",
] as const;
export const BrowserAutomationCommandNameSchema = z.enum(BROWSER_AUTOMATION_COMMAND_NAMES);
@@ -55,6 +59,8 @@ const BrowserAutomationTabTargetSchema = z
.strict();
const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/);
const BrowserAutomationMouseButtonSchema = z.enum(["left", "right", "middle"]);
const BrowserAutomationInputModifierSchema = z.enum(["Alt", "Control", "Meta", "Shift"]);
const BrowserAutomationHttpUrlSchema = z
.string()
.url()
@@ -87,6 +93,9 @@ export const BrowserAutomationClickCommandSchema = z.object({
command: z.literal("click"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
button: BrowserAutomationMouseButtonSchema.default("left"),
doubleClick: z.boolean().default(false),
modifiers: z.array(BrowserAutomationInputModifierSchema).default([]),
}),
});
@@ -192,6 +201,36 @@ export const BrowserAutomationLogsCommandSchema = z.object({
}),
});
export const BrowserAutomationEvaluateCommandSchema = z.object({
command: z.literal("evaluate"),
args: BrowserAutomationTabTargetSchema.extend({
function: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
}),
});
export const BrowserAutomationScrollCommandSchema = z.object({
command: z.literal("scroll"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema.optional(),
deltaX: z.number(),
deltaY: z.number(),
}),
});
export const BrowserAutomationResizeCommandSchema = z.object({
command: z.literal("resize"),
args: BrowserAutomationTabTargetSchema.extend({
width: z.number().int().positive(),
height: z.number().int().positive(),
}),
});
export const BrowserAutomationCloseTabCommandSchema = z.object({
command: z.literal("close_tab"),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsCommandSchema,
BrowserAutomationNewTabCommandSchema,
@@ -211,6 +250,10 @@ export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
BrowserAutomationHoverCommandSchema,
BrowserAutomationDragCommandSchema,
BrowserAutomationLogsCommandSchema,
BrowserAutomationEvaluateCommandSchema,
BrowserAutomationScrollCommandSchema,
BrowserAutomationResizeCommandSchema,
BrowserAutomationCloseTabCommandSchema,
]);
export const BrowserAutomationTabInfoSchema = z.object({
@@ -236,14 +279,15 @@ export const BrowserAutomationNewTabResultSchema = z.object({
url: z.string().min(1),
});
export const BrowserAutomationSnapshotElementSchema = z.object({
ref: z.string().regex(/^@e\d+$/),
role: z.string(),
tagName: z.string(),
text: z.string(),
selector: z.string(),
attributes: z.record(z.string(), z.string()).default({}),
});
export const BrowserAutomationSnapshotStatsSchema = z
.object({
nodeCount: z.number().int().nonnegative(),
refCount: z.number().int().nonnegative(),
textLength: z.number().int().nonnegative(),
iframeCount: z.number().int().nonnegative().optional(),
maxDepth: z.number().int().nonnegative().optional(),
})
.strict();
export const BrowserAutomationSnapshotResultSchema = z.object({
command: z.literal("snapshot"),
@@ -251,13 +295,18 @@ export const BrowserAutomationSnapshotResultSchema = z.object({
workspaceId: z.string().min(1).optional(),
url: z.string(),
title: z.string(),
elements: z.array(BrowserAutomationSnapshotElementSchema),
format: z.literal("aria-yaml"),
snapshot: z.string(),
truncated: z.boolean(),
stats: BrowserAutomationSnapshotStatsSchema,
});
export const BrowserAutomationClickResultSchema = z.object({
command: z.literal("click"),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
x: z.number().optional(),
y: z.number().optional(),
});
export const BrowserAutomationFillResultSchema = z.object({
@@ -276,6 +325,8 @@ export const BrowserAutomationTypeResultSchema = z.object({
command: z.literal("type"),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema.optional(),
x: z.number().optional(),
y: z.number().optional(),
});
export const BrowserAutomationKeypressResultSchema = z.object({
@@ -283,6 +334,8 @@ export const BrowserAutomationKeypressResultSchema = z.object({
browserId: BrowserAutomationBrowserIdSchema,
key: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
x: z.number().optional(),
y: z.number().optional(),
});
export const BrowserAutomationNavigateResultSchema = z.object({
@@ -333,6 +386,8 @@ export const BrowserAutomationHoverResultSchema = z.object({
command: z.literal("hover"),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema,
x: z.number().optional(),
y: z.number().optional(),
});
export const BrowserAutomationDragResultSchema = z.object({
@@ -340,6 +395,10 @@ export const BrowserAutomationDragResultSchema = z.object({
browserId: BrowserAutomationBrowserIdSchema,
sourceRef: BrowserAutomationRefSchema,
targetRef: BrowserAutomationRefSchema,
sourceX: z.number().optional(),
sourceY: z.number().optional(),
targetX: z.number().optional(),
targetY: z.number().optional(),
});
export const BrowserAutomationConsoleLogEntrySchema = z.object({
@@ -367,6 +426,35 @@ export const BrowserAutomationLogsResultSchema = z.object({
network: z.array(BrowserAutomationNetworkLogEntrySchema),
});
export const BrowserAutomationEvaluateResultSchema = z.object({
command: z.literal("evaluate"),
browserId: BrowserAutomationBrowserIdSchema,
resultJson: z.string(),
truncated: z.boolean(),
});
export const BrowserAutomationScrollResultSchema = z.object({
command: z.literal("scroll"),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema.optional(),
deltaX: z.number(),
deltaY: z.number(),
x: z.number().optional(),
y: z.number().optional(),
});
export const BrowserAutomationResizeResultSchema = z.object({
command: z.literal("resize"),
browserId: BrowserAutomationBrowserIdSchema,
width: z.number().int().positive(),
height: z.number().int().positive(),
});
export const BrowserAutomationCloseTabResultSchema = z.object({
command: z.literal("close_tab"),
browserId: BrowserAutomationBrowserIdSchema,
});
export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsResultSchema,
BrowserAutomationNewTabResultSchema,
@@ -386,6 +474,10 @@ export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
BrowserAutomationHoverResultSchema,
BrowserAutomationDragResultSchema,
BrowserAutomationLogsResultSchema,
BrowserAutomationEvaluateResultSchema,
BrowserAutomationScrollResultSchema,
BrowserAutomationResizeResultSchema,
BrowserAutomationCloseTabResultSchema,
]);
export const BrowserAutomationErrorSchema = z.object({
@@ -394,6 +486,15 @@ export const BrowserAutomationErrorSchema = z.object({
retryable: z.boolean().default(false),
});
export const BrowserAutomationDialogEventSchema = z.object({
type: z.enum(["alert", "confirm", "prompt", "beforeunload"]),
message: z.string(),
defaultValue: z.string().optional(),
action: z.enum(["accepted", "dismissed"]),
promptText: z.string().optional(),
timestamp: z.number(),
});
export const BrowserAutomationExecuteRequestSchema = z
.object({
type: z.literal("browser.automation.execute.request"),
@@ -412,11 +513,13 @@ export const BrowserAutomationExecuteResponseSchema = z.object({
requestId: z.string().min(1),
ok: z.literal(true),
result: BrowserAutomationResultSchema,
dialogs: z.array(BrowserAutomationDialogEventSchema).optional(),
}),
z.object({
requestId: z.string().min(1),
ok: z.literal(false),
error: BrowserAutomationErrorSchema,
dialogs: z.array(BrowserAutomationDialogEventSchema).optional(),
}),
]),
});
@@ -431,6 +534,7 @@ export type BrowserAutomationConsoleLogEntry = z.infer<
export type BrowserAutomationNetworkLogEntry = z.infer<
typeof BrowserAutomationNetworkLogEntrySchema
>;
export type BrowserAutomationDialogEvent = z.infer<typeof BrowserAutomationDialogEventSchema>;
export type BrowserAutomationExecuteRequest = z.infer<typeof BrowserAutomationExecuteRequestSchema>;
export type BrowserAutomationExecuteResponse = z.infer<
typeof BrowserAutomationExecuteResponseSchema
packages/protocol/src/messages.browser-automation.test.ts
+22
View File
@@ -100,6 +100,28 @@ describe("browser automation protocol integration", () => {
});
});
test("browser host capability accepts new tool commands as supported commands", () => {
expect(
WSHelloMessageSchema.parse({
type: "hello",
clientId: "client-1",
clientType: "mobile",
protocolVersion: 1,
capabilities: {
[CLIENT_CAPS.browserHost]: {
supportedCommands: ["evaluate", "scroll", "resize", "close_tab"],
hostKind: "desktop app",
},
},
}).capabilities,
).toMatchObject({
[CLIENT_CAPS.browserHost]: {
supportedCommands: ["evaluate", "scroll", "resize", "close_tab"],
hostKind: "desktop app",
},
});
});
test("hello remains valid when no browser host capability is advertised", () => {
expect(
WSHelloMessageSchema.parse({