Add opt-in browser tools for desktop tabs
* Add opt-in browser tools for desktop tabs Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs. * Fix browser tools CI expectations * Address browser tools review findings * Restrict browser file automation paths * Fix browser upload test on Windows * Harden browser navigation inputs * Make browser tools create usable tabs * Update browser MCP empty-state test * Fail browser tab creation when registration times out * Fix browser screenshots for agents * Hide disabled browser tools from agents * Address browser tools architecture review * Replace browser tools review tests * Wrap browser tab registration errors * Mock Expo Router in app unit tests * Handle invalid browser automation requests * Return browser failure on desktop disconnect * Update browser disconnect websocket test * Relax browser timeout polling test * Handle invalid browser responses * Return browser failure when send fails * Remove local diagnostics and fixture paths * Fix dev service home fallback * Use worktree home for dev services * Use managed daemon in desktop dev * fix(browser): keep agent tabs addressable Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution. * refactor(browser): register tools through catalog Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter. * fix(settings): translate browser tools host error
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,750 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import {
|
||||
BrowserAutomationExecuteRequestSchema,
|
||||
BrowserAutomationExecuteResponseSchema,
|
||||
} from "./rpc-schemas.js";
|
||||
|
||||
describe("browser automation execute RPC schemas", () => {
|
||||
test("rejects navigate and download commands for non-http URLs", () => {
|
||||
for (const command of ["navigate", "download"] as const) {
|
||||
expect(() =>
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: `req-${command}`,
|
||||
command: {
|
||||
command,
|
||||
args: { url: "file:///tmp/secret.txt" },
|
||||
},
|
||||
}),
|
||||
).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test("parses list tabs requests with top-level correlation and typed command args", () => {
|
||||
const parsed = BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
workspaceId: "workspace-1",
|
||||
command: {
|
||||
command: "list_tabs",
|
||||
args: { workspaceId: "workspace-1" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(parsed).toEqual({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
workspaceId: "workspace-1",
|
||||
command: {
|
||||
command: "list_tabs",
|
||||
args: { workspaceId: "workspace-1" },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("parses new tab requests and responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-new-tab",
|
||||
workspaceId: "workspace-1",
|
||||
command: {
|
||||
command: "new_tab",
|
||||
args: { workspaceId: "workspace-1", url: "https://example.com" },
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "new_tab",
|
||||
args: { workspaceId: "workspace-1", url: "https://example.com" },
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-new-tab",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
},
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-new-tab",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("parses page info responses with result data under payload", () => {
|
||||
const parsed = BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "page_info",
|
||||
tab: {
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(parsed.payload).toEqual({
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "page_info",
|
||||
tab: {
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
isActive: false,
|
||||
isLoading: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("parses snapshot requests and ref-bearing snapshot responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-snapshot",
|
||||
workspaceId: "workspace-1",
|
||||
command: {
|
||||
command: "snapshot",
|
||||
args: { workspaceId: "workspace-1", browserId: "browser-1" },
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-snapshot",
|
||||
workspaceId: "workspace-1",
|
||||
command: {
|
||||
command: "snapshot",
|
||||
args: { workspaceId: "workspace-1", browserId: "browser-1" },
|
||||
},
|
||||
});
|
||||
|
||||
const parsed = BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-snapshot",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "snapshot",
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com/form",
|
||||
title: "Fixture",
|
||||
elements: [
|
||||
{
|
||||
ref: "@e1",
|
||||
role: "textbox",
|
||||
tagName: "input",
|
||||
text: "Name",
|
||||
selector: "#name",
|
||||
attributes: { id: "name", type: "text" },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(parsed.payload).toEqual({
|
||||
requestId: "req-snapshot",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "snapshot",
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com/form",
|
||||
title: "Fixture",
|
||||
elements: [
|
||||
{
|
||||
ref: "@e1",
|
||||
role: "textbox",
|
||||
tagName: "input",
|
||||
text: "Name",
|
||||
selector: "#name",
|
||||
attributes: { id: "name", type: "text" },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("parses click and fill ref commands", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-click",
|
||||
command: {
|
||||
command: "click",
|
||||
args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" },
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "click",
|
||||
args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" },
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-fill",
|
||||
command: {
|
||||
command: "fill",
|
||||
args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" },
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "fill",
|
||||
args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" },
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-fill",
|
||||
ok: true,
|
||||
result: { command: "fill", browserId: "browser-1", ref: "@e2" },
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-fill",
|
||||
ok: true,
|
||||
result: { command: "fill", browserId: "browser-1", ref: "@e2" },
|
||||
});
|
||||
});
|
||||
|
||||
test("parses wait text commands and responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-wait",
|
||||
command: {
|
||||
command: "wait",
|
||||
args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 },
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "wait",
|
||||
args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 },
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-wait",
|
||||
ok: true,
|
||||
result: { command: "wait", browserId: "browser-1", matched: "text" },
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-wait",
|
||||
ok: true,
|
||||
result: { command: "wait", browserId: "browser-1", matched: "text" },
|
||||
});
|
||||
});
|
||||
|
||||
test("parses set background commands and responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-bg",
|
||||
command: {
|
||||
command: "set_background",
|
||||
args: { workspaceId: "workspace-1", color: "red" },
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "set_background",
|
||||
args: { workspaceId: "workspace-1", color: "red" },
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-bg",
|
||||
ok: true,
|
||||
result: { command: "set_background", browserId: "browser-1", color: "red" },
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-bg",
|
||||
ok: true,
|
||||
result: { command: "set_background", browserId: "browser-1", color: "red" },
|
||||
});
|
||||
});
|
||||
|
||||
test("parses type and keypress commands", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-type",
|
||||
command: { command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } },
|
||||
}).command,
|
||||
).toEqual({ command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-keypress",
|
||||
command: { command: "keypress", args: { browserId: "browser-1", key: "Enter" } },
|
||||
}).command,
|
||||
).toEqual({ command: "keypress", args: { browserId: "browser-1", key: "Enter" } });
|
||||
});
|
||||
|
||||
test("parses navigation commands", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-nav",
|
||||
command: {
|
||||
command: "navigate",
|
||||
args: { browserId: "browser-1", url: "https://example.com/next" },
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "navigate",
|
||||
args: { browserId: "browser-1", url: "https://example.com/next" },
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-nav",
|
||||
ok: true,
|
||||
result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" },
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-nav",
|
||||
ok: true,
|
||||
result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" },
|
||||
});
|
||||
});
|
||||
|
||||
test("parses screenshot responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-shot",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "screenshot",
|
||||
browserId: "browser-1",
|
||||
mimeType: "image/png",
|
||||
dataBase64: "iVBORw0KGgo=",
|
||||
width: 100,
|
||||
height: 50,
|
||||
},
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-shot",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "screenshot",
|
||||
browserId: "browser-1",
|
||||
mimeType: "image/png",
|
||||
dataBase64: "iVBORw0KGgo=",
|
||||
width: 100,
|
||||
height: 50,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("parses form control commands and responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-focus",
|
||||
command: { command: "focus", args: { browserId: "browser-1", ref: "@e1" } },
|
||||
}).command,
|
||||
).toEqual({ command: "focus", args: { browserId: "browser-1", ref: "@e1" } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-clear",
|
||||
command: { command: "clear", args: { browserId: "browser-1", ref: "@e1" } },
|
||||
}).command,
|
||||
).toEqual({ command: "clear", args: { browserId: "browser-1", ref: "@e1" } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-check",
|
||||
command: { command: "check", args: { browserId: "browser-1", ref: "@e2" } },
|
||||
}).command,
|
||||
).toEqual({ command: "check", args: { browserId: "browser-1", ref: "@e2", checked: true } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-select",
|
||||
command: { command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } },
|
||||
}).command,
|
||||
).toEqual({ command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-hover",
|
||||
command: { command: "hover", args: { browserId: "browser-1", ref: "@e4" } },
|
||||
}).command,
|
||||
).toEqual({ command: "hover", args: { browserId: "browser-1", ref: "@e4" } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-drag",
|
||||
command: {
|
||||
command: "drag",
|
||||
args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" },
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "drag",
|
||||
args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" },
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-select",
|
||||
ok: true,
|
||||
result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" },
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-select",
|
||||
ok: true,
|
||||
result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" },
|
||||
});
|
||||
});
|
||||
|
||||
test("parses browser log commands and responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-logs",
|
||||
command: { command: "logs", args: { browserId: "browser-1" } },
|
||||
}).command,
|
||||
).toEqual({ command: "logs", args: { browserId: "browser-1", maxEntries: 50 } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-logs",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "logs",
|
||||
browserId: "browser-1",
|
||||
console: [{ level: "info", message: "ready", timestamp: 10 }],
|
||||
network: [
|
||||
{
|
||||
url: "https://example.com/app.js",
|
||||
type: "script",
|
||||
startTime: 1,
|
||||
duration: 2,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-logs",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "logs",
|
||||
browserId: "browser-1",
|
||||
console: [{ level: "info", message: "ready", timestamp: 10 }],
|
||||
network: [
|
||||
{
|
||||
url: "https://example.com/app.js",
|
||||
type: "script",
|
||||
startTime: 1,
|
||||
duration: 2,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("parses browser storage commands and responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-storage",
|
||||
command: { command: "storage", args: { browserId: "browser-1" } },
|
||||
}).command,
|
||||
).toEqual({ command: "storage", args: { browserId: "browser-1" } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-storage",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "storage",
|
||||
browserId: "browser-1",
|
||||
url: "https://example.com",
|
||||
cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }],
|
||||
localStorage: [{ key: "token", value: "abc" }],
|
||||
sessionStorage: [{ key: "tab", value: "1" }],
|
||||
},
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-storage",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "storage",
|
||||
browserId: "browser-1",
|
||||
url: "https://example.com",
|
||||
cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }],
|
||||
localStorage: [{ key: "token", value: "abc" }],
|
||||
sessionStorage: [{ key: "tab", value: "1" }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("parses browser environment commands and responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-environment",
|
||||
command: {
|
||||
command: "environment",
|
||||
args: {
|
||||
browserId: "browser-1",
|
||||
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
|
||||
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
|
||||
},
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "environment",
|
||||
args: {
|
||||
browserId: "browser-1",
|
||||
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
|
||||
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-environment",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "environment",
|
||||
browserId: "browser-1",
|
||||
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
|
||||
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
|
||||
},
|
||||
},
|
||||
}).payload,
|
||||
).toEqual({
|
||||
requestId: "req-environment",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "environment",
|
||||
browserId: "browser-1",
|
||||
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
|
||||
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("parses browser full-page screenshot and PDF commands and responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-full-page",
|
||||
command: { command: "full_page_screenshot", args: { browserId: "browser-1" } },
|
||||
}).command,
|
||||
).toEqual({ command: "full_page_screenshot", args: { browserId: "browser-1" } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-pdf",
|
||||
command: { command: "pdf", args: { browserId: "browser-1" } },
|
||||
}).command,
|
||||
).toEqual({ command: "pdf", args: { browserId: "browser-1", printBackground: true } });
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-full-page",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "full_page_screenshot",
|
||||
browserId: "browser-1",
|
||||
mimeType: "image/png",
|
||||
dataBase64: "iVBORw0KGgo=",
|
||||
width: 390,
|
||||
height: 1200,
|
||||
},
|
||||
},
|
||||
}).payload.result,
|
||||
).toEqual({
|
||||
command: "full_page_screenshot",
|
||||
browserId: "browser-1",
|
||||
mimeType: "image/png",
|
||||
dataBase64: "iVBORw0KGgo=",
|
||||
width: 390,
|
||||
height: 1200,
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-pdf",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "pdf",
|
||||
browserId: "browser-1",
|
||||
mimeType: "application/pdf",
|
||||
dataBase64: "JVBERi0xLjQ=",
|
||||
},
|
||||
},
|
||||
}).payload.result,
|
||||
).toEqual({
|
||||
command: "pdf",
|
||||
browserId: "browser-1",
|
||||
mimeType: "application/pdf",
|
||||
dataBase64: "JVBERi0xLjQ=",
|
||||
});
|
||||
});
|
||||
|
||||
test("parses browser download and upload commands and responses", () => {
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-download",
|
||||
command: {
|
||||
command: "download",
|
||||
args: { browserId: "browser-1", url: "https://example.com/file.txt" },
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "download",
|
||||
args: { browserId: "browser-1", url: "https://example.com/file.txt" },
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteRequestSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-upload",
|
||||
command: {
|
||||
command: "upload",
|
||||
args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] },
|
||||
},
|
||||
}).command,
|
||||
).toEqual({
|
||||
command: "upload",
|
||||
args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] },
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-download",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "download",
|
||||
browserId: "browser-1",
|
||||
url: "https://example.com/file.txt",
|
||||
filePath: "/tmp/file.txt",
|
||||
totalBytes: 5,
|
||||
state: "completed",
|
||||
},
|
||||
},
|
||||
}).payload.result,
|
||||
).toEqual({
|
||||
command: "download",
|
||||
browserId: "browser-1",
|
||||
url: "https://example.com/file.txt",
|
||||
filePath: "/tmp/file.txt",
|
||||
totalBytes: 5,
|
||||
state: "completed",
|
||||
});
|
||||
|
||||
expect(
|
||||
BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-upload",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "upload",
|
||||
browserId: "browser-1",
|
||||
ref: "@e1",
|
||||
filePaths: ["/tmp/file.txt"],
|
||||
},
|
||||
},
|
||||
}).payload.result,
|
||||
).toEqual({
|
||||
command: "upload",
|
||||
browserId: "browser-1",
|
||||
ref: "@e1",
|
||||
filePaths: ["/tmp/file.txt"],
|
||||
});
|
||||
});
|
||||
|
||||
test("parses stable model-actionable error responses", () => {
|
||||
const parsed = BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_no_desktop",
|
||||
message: "No desktop browser automation client is connected.",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(parsed.payload).toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_no_desktop",
|
||||
message: "No desktop browser automation client is connected.",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,599 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const BrowserAutomationErrorCodeSchema = z.enum([
|
||||
"browser_disabled",
|
||||
"browser_no_desktop",
|
||||
"browser_no_tab",
|
||||
"browser_tab_not_found",
|
||||
"browser_tab_closed",
|
||||
"browser_timeout",
|
||||
"browser_denied",
|
||||
"browser_unsupported",
|
||||
"browser_stale_ref",
|
||||
"browser_unknown_error",
|
||||
]);
|
||||
|
||||
const BrowserAutomationTabTargetSchema = z.object({
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
browserId: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/);
|
||||
const BrowserAutomationHttpUrlSchema = z
|
||||
.string()
|
||||
.url()
|
||||
.refine((value) => {
|
||||
const protocol = new URL(value).protocol;
|
||||
return protocol === "http:" || protocol === "https:";
|
||||
}, "URL must use http or https");
|
||||
|
||||
export const BrowserAutomationListTabsCommandSchema = z.object({
|
||||
command: z.literal("list_tabs"),
|
||||
args: z
|
||||
.object({
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
})
|
||||
.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationNewTabCommandSchema = z.object({
|
||||
command: z.literal("new_tab"),
|
||||
args: z
|
||||
.object({
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
url: BrowserAutomationHttpUrlSchema.optional(),
|
||||
})
|
||||
.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationPageInfoCommandSchema = z.object({
|
||||
command: z.literal("page_info"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSnapshotCommandSchema = z.object({
|
||||
command: z.literal("snapshot"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationClickCommandSchema = z.object({
|
||||
command: z.literal("click"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFillCommandSchema = z.object({
|
||||
command: z.literal("fill"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
value: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationWaitCommandSchema = z.object({
|
||||
command: z.literal("wait"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
text: z.string().min(1).optional(),
|
||||
url: z.string().min(1).optional(),
|
||||
timeoutMs: z.number().int().positive().max(30_000).optional(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationTypeCommandSchema = z.object({
|
||||
command: z.literal("type"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
text: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationKeypressCommandSchema = z.object({
|
||||
command: z.literal("keypress"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
key: z.string().min(1),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationNavigateCommandSchema = z.object({
|
||||
command: z.literal("navigate"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
url: BrowserAutomationHttpUrlSchema,
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationBackCommandSchema = z.object({
|
||||
command: z.literal("back"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationForwardCommandSchema = z.object({
|
||||
command: z.literal("forward"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationReloadCommandSchema = z.object({
|
||||
command: z.literal("reload"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationScreenshotCommandSchema = z.object({
|
||||
command: z.literal("screenshot"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({
|
||||
command: z.literal("full_page_screenshot"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationPdfCommandSchema = z.object({
|
||||
command: z.literal("pdf"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
landscape: z.boolean().optional(),
|
||||
printBackground: z.boolean().default(true),
|
||||
}).default({ printBackground: true }),
|
||||
});
|
||||
|
||||
export const BrowserAutomationDownloadCommandSchema = z.object({
|
||||
command: z.literal("download"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
url: BrowserAutomationHttpUrlSchema,
|
||||
fileName: z.string().min(1).optional(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationUploadCommandSchema = z.object({
|
||||
command: z.literal("upload"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
filePaths: z.array(z.string().min(1)).min(1),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFocusCommandSchema = z.object({
|
||||
command: z.literal("focus"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationClearCommandSchema = z.object({
|
||||
command: z.literal("clear"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationCheckCommandSchema = z.object({
|
||||
command: z.literal("check"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
checked: z.boolean().default(true),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSelectCommandSchema = z.object({
|
||||
command: z.literal("select"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
value: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationHoverCommandSchema = z.object({
|
||||
command: z.literal("hover"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationDragCommandSchema = z.object({
|
||||
command: z.literal("drag"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
sourceRef: BrowserAutomationRefSchema,
|
||||
targetRef: BrowserAutomationRefSchema,
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationLogsCommandSchema = z.object({
|
||||
command: z.literal("logs"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
maxEntries: z.number().int().positive().max(200).default(50),
|
||||
}).default({ maxEntries: 50 }),
|
||||
});
|
||||
|
||||
export const BrowserAutomationStorageCommandSchema = z.object({
|
||||
command: z.literal("storage"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
const BrowserAutomationViewportInputSchema = z.object({
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
deviceScaleFactor: z.number().positive().optional(),
|
||||
});
|
||||
|
||||
const BrowserAutomationGeolocationInputSchema = z.object({
|
||||
latitude: z.number().min(-90).max(90),
|
||||
longitude: z.number().min(-180).max(180),
|
||||
accuracy: z.number().positive().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationEnvironmentCommandSchema = z.object({
|
||||
command: z.literal("environment"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
viewport: BrowserAutomationViewportInputSchema.optional(),
|
||||
geolocation: BrowserAutomationGeolocationInputSchema.optional(),
|
||||
}).default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSetBackgroundCommandSchema = z.object({
|
||||
command: z.literal("set_background"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
color: z.string().min(1),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationListTabsCommandSchema,
|
||||
BrowserAutomationNewTabCommandSchema,
|
||||
BrowserAutomationPageInfoCommandSchema,
|
||||
BrowserAutomationSnapshotCommandSchema,
|
||||
BrowserAutomationClickCommandSchema,
|
||||
BrowserAutomationFillCommandSchema,
|
||||
BrowserAutomationWaitCommandSchema,
|
||||
BrowserAutomationTypeCommandSchema,
|
||||
BrowserAutomationKeypressCommandSchema,
|
||||
BrowserAutomationNavigateCommandSchema,
|
||||
BrowserAutomationBackCommandSchema,
|
||||
BrowserAutomationForwardCommandSchema,
|
||||
BrowserAutomationReloadCommandSchema,
|
||||
BrowserAutomationScreenshotCommandSchema,
|
||||
BrowserAutomationFullPageScreenshotCommandSchema,
|
||||
BrowserAutomationPdfCommandSchema,
|
||||
BrowserAutomationDownloadCommandSchema,
|
||||
BrowserAutomationUploadCommandSchema,
|
||||
BrowserAutomationFocusCommandSchema,
|
||||
BrowserAutomationClearCommandSchema,
|
||||
BrowserAutomationCheckCommandSchema,
|
||||
BrowserAutomationSelectCommandSchema,
|
||||
BrowserAutomationHoverCommandSchema,
|
||||
BrowserAutomationDragCommandSchema,
|
||||
BrowserAutomationLogsCommandSchema,
|
||||
BrowserAutomationStorageCommandSchema,
|
||||
BrowserAutomationEnvironmentCommandSchema,
|
||||
BrowserAutomationSetBackgroundCommandSchema,
|
||||
]);
|
||||
|
||||
export const BrowserAutomationTabInfoSchema = z.object({
|
||||
browserId: z.string().min(1),
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
url: z.string(),
|
||||
title: z.string(),
|
||||
isActive: z.boolean().default(false),
|
||||
isLoading: z.boolean().default(false),
|
||||
canGoBack: z.boolean().optional(),
|
||||
canGoForward: z.boolean().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationListTabsResultSchema = z.object({
|
||||
command: z.literal("list_tabs"),
|
||||
tabs: z.array(BrowserAutomationTabInfoSchema),
|
||||
});
|
||||
|
||||
export const BrowserAutomationNewTabResultSchema = z.object({
|
||||
command: z.literal("new_tab"),
|
||||
browserId: z.string().min(1),
|
||||
workspaceId: z.string().min(1),
|
||||
url: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationPageInfoResultSchema = z.object({
|
||||
command: z.literal("page_info"),
|
||||
tab: BrowserAutomationTabInfoSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationSnapshotElementSchema = z.object({
|
||||
ref: z.string().regex(/^@e\d+$/),
|
||||
role: z.string(),
|
||||
tagName: z.string(),
|
||||
text: z.string(),
|
||||
selector: z.string(),
|
||||
attributes: z.record(z.string(), z.string()).default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSnapshotResultSchema = z.object({
|
||||
command: z.literal("snapshot"),
|
||||
browserId: z.string().min(1),
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
url: z.string(),
|
||||
title: z.string(),
|
||||
elements: z.array(BrowserAutomationSnapshotElementSchema),
|
||||
});
|
||||
|
||||
export const BrowserAutomationClickResultSchema = z.object({
|
||||
command: z.literal("click"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationFillResultSchema = z.object({
|
||||
command: z.literal("fill"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationWaitResultSchema = z.object({
|
||||
command: z.literal("wait"),
|
||||
browserId: z.string().min(1),
|
||||
matched: z.enum(["text", "url"]),
|
||||
});
|
||||
|
||||
export const BrowserAutomationTypeResultSchema = z.object({
|
||||
command: z.literal("type"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationKeypressResultSchema = z.object({
|
||||
command: z.literal("keypress"),
|
||||
browserId: z.string().min(1),
|
||||
key: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationNavigateResultSchema = z.object({
|
||||
command: z.literal("navigate"),
|
||||
browserId: z.string().min(1),
|
||||
url: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationBackResultSchema = z.object({
|
||||
command: z.literal("back"),
|
||||
browserId: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationForwardResultSchema = z.object({
|
||||
command: z.literal("forward"),
|
||||
browserId: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationReloadResultSchema = z.object({
|
||||
command: z.literal("reload"),
|
||||
browserId: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationScreenshotResultSchema = z.object({
|
||||
command: z.literal("screenshot"),
|
||||
browserId: z.string().min(1),
|
||||
mimeType: z.literal("image/png"),
|
||||
dataBase64: z.string().min(1),
|
||||
width: z.number().int().nonnegative(),
|
||||
height: z.number().int().nonnegative(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFullPageScreenshotResultSchema = z.object({
|
||||
command: z.literal("full_page_screenshot"),
|
||||
browserId: z.string().min(1),
|
||||
mimeType: z.literal("image/png"),
|
||||
dataBase64: z.string().min(1),
|
||||
width: z.number().int().nonnegative(),
|
||||
height: z.number().int().nonnegative(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationPdfResultSchema = z.object({
|
||||
command: z.literal("pdf"),
|
||||
browserId: z.string().min(1),
|
||||
mimeType: z.literal("application/pdf"),
|
||||
dataBase64: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationDownloadResultSchema = z.object({
|
||||
command: z.literal("download"),
|
||||
browserId: z.string().min(1),
|
||||
url: z.string().min(1),
|
||||
filePath: z.string().min(1),
|
||||
totalBytes: z.number().int().nonnegative().optional(),
|
||||
state: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationUploadResultSchema = z.object({
|
||||
command: z.literal("upload"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
filePaths: z.array(z.string().min(1)).min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFocusResultSchema = z.object({
|
||||
command: z.literal("focus"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationClearResultSchema = z.object({
|
||||
command: z.literal("clear"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationCheckResultSchema = z.object({
|
||||
command: z.literal("check"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
checked: z.boolean(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSelectResultSchema = z.object({
|
||||
command: z.literal("select"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
value: z.string(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationHoverResultSchema = z.object({
|
||||
command: z.literal("hover"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationDragResultSchema = z.object({
|
||||
command: z.literal("drag"),
|
||||
browserId: z.string().min(1),
|
||||
sourceRef: BrowserAutomationRefSchema,
|
||||
targetRef: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationConsoleLogEntrySchema = z.object({
|
||||
level: z.string(),
|
||||
message: z.string(),
|
||||
source: z.string().optional(),
|
||||
line: z.number().int().optional(),
|
||||
timestamp: z.number(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationNetworkLogEntrySchema = z.object({
|
||||
url: z.string(),
|
||||
method: z.string().optional(),
|
||||
status: z.number().int().optional(),
|
||||
type: z.string().optional(),
|
||||
startTime: z.number(),
|
||||
duration: z.number(),
|
||||
transferSize: z.number().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationLogsResultSchema = z.object({
|
||||
command: z.literal("logs"),
|
||||
browserId: z.string().min(1),
|
||||
console: z.array(BrowserAutomationConsoleLogEntrySchema),
|
||||
network: z.array(BrowserAutomationNetworkLogEntrySchema),
|
||||
});
|
||||
|
||||
export const BrowserAutomationCookieEntrySchema = z.object({
|
||||
name: z.string(),
|
||||
value: z.string(),
|
||||
domain: z.string().optional(),
|
||||
path: z.string().optional(),
|
||||
secure: z.boolean().optional(),
|
||||
httpOnly: z.boolean().optional(),
|
||||
expirationDate: z.number().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationStorageEntrySchema = z.object({
|
||||
key: z.string(),
|
||||
value: z.string(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationStorageResultSchema = z.object({
|
||||
command: z.literal("storage"),
|
||||
browserId: z.string().min(1),
|
||||
url: z.string(),
|
||||
cookies: z.array(BrowserAutomationCookieEntrySchema),
|
||||
localStorage: z.array(BrowserAutomationStorageEntrySchema),
|
||||
sessionStorage: z.array(BrowserAutomationStorageEntrySchema),
|
||||
});
|
||||
|
||||
export const BrowserAutomationViewportResultSchema = z.object({
|
||||
width: z.number().int().nonnegative(),
|
||||
height: z.number().int().nonnegative(),
|
||||
deviceScaleFactor: z.number().positive(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationGeolocationResultSchema = z.object({
|
||||
latitude: z.number(),
|
||||
longitude: z.number(),
|
||||
accuracy: z.number(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationEnvironmentResultSchema = z.object({
|
||||
command: z.literal("environment"),
|
||||
browserId: z.string().min(1),
|
||||
viewport: BrowserAutomationViewportResultSchema,
|
||||
geolocation: BrowserAutomationGeolocationResultSchema.optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSetBackgroundResultSchema = z.object({
|
||||
command: z.literal("set_background"),
|
||||
browserId: z.string().min(1),
|
||||
color: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationListTabsResultSchema,
|
||||
BrowserAutomationNewTabResultSchema,
|
||||
BrowserAutomationPageInfoResultSchema,
|
||||
BrowserAutomationSnapshotResultSchema,
|
||||
BrowserAutomationClickResultSchema,
|
||||
BrowserAutomationFillResultSchema,
|
||||
BrowserAutomationWaitResultSchema,
|
||||
BrowserAutomationTypeResultSchema,
|
||||
BrowserAutomationKeypressResultSchema,
|
||||
BrowserAutomationNavigateResultSchema,
|
||||
BrowserAutomationBackResultSchema,
|
||||
BrowserAutomationForwardResultSchema,
|
||||
BrowserAutomationReloadResultSchema,
|
||||
BrowserAutomationScreenshotResultSchema,
|
||||
BrowserAutomationFullPageScreenshotResultSchema,
|
||||
BrowserAutomationPdfResultSchema,
|
||||
BrowserAutomationDownloadResultSchema,
|
||||
BrowserAutomationUploadResultSchema,
|
||||
BrowserAutomationFocusResultSchema,
|
||||
BrowserAutomationClearResultSchema,
|
||||
BrowserAutomationCheckResultSchema,
|
||||
BrowserAutomationSelectResultSchema,
|
||||
BrowserAutomationHoverResultSchema,
|
||||
BrowserAutomationDragResultSchema,
|
||||
BrowserAutomationLogsResultSchema,
|
||||
BrowserAutomationStorageResultSchema,
|
||||
BrowserAutomationEnvironmentResultSchema,
|
||||
BrowserAutomationSetBackgroundResultSchema,
|
||||
]);
|
||||
|
||||
export const BrowserAutomationErrorSchema = z.object({
|
||||
code: BrowserAutomationErrorCodeSchema,
|
||||
message: z.string().min(1),
|
||||
retryable: z.boolean().default(false),
|
||||
});
|
||||
|
||||
export const BrowserAutomationExecuteRequestSchema = z.object({
|
||||
type: z.literal("browser.automation.execute.request"),
|
||||
requestId: z.string().min(1),
|
||||
agentId: z.string().min(1).optional(),
|
||||
cwd: z.string().min(1).optional(),
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
browserId: z.string().min(1).optional(),
|
||||
command: BrowserAutomationCommandSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationExecuteResponseSchema = z.object({
|
||||
type: z.literal("browser.automation.execute.response"),
|
||||
payload: z.discriminatedUnion("ok", [
|
||||
z.object({
|
||||
requestId: z.string().min(1),
|
||||
ok: z.literal(true),
|
||||
result: BrowserAutomationResultSchema,
|
||||
}),
|
||||
z.object({
|
||||
requestId: z.string().min(1),
|
||||
ok: z.literal(false),
|
||||
error: BrowserAutomationErrorSchema,
|
||||
}),
|
||||
]),
|
||||
});
|
||||
|
||||
export type BrowserAutomationErrorCode = z.infer<typeof BrowserAutomationErrorCodeSchema>;
|
||||
export type BrowserAutomationCommand = z.infer<typeof BrowserAutomationCommandSchema>;
|
||||
export type BrowserAutomationResult = z.infer<typeof BrowserAutomationResultSchema>;
|
||||
export type BrowserAutomationConsoleLogEntry = z.infer<
|
||||
typeof BrowserAutomationConsoleLogEntrySchema
|
||||
>;
|
||||
export type BrowserAutomationNetworkLogEntry = z.infer<
|
||||
typeof BrowserAutomationNetworkLogEntrySchema
|
||||
>;
|
||||
export type BrowserAutomationCookieEntry = z.infer<typeof BrowserAutomationCookieEntrySchema>;
|
||||
export type BrowserAutomationStorageEntry = z.infer<typeof BrowserAutomationStorageEntrySchema>;
|
||||
export type BrowserAutomationExecuteRequest = z.infer<typeof BrowserAutomationExecuteRequestSchema>;
|
||||
export type BrowserAutomationExecuteResponse = z.infer<
|
||||
typeof BrowserAutomationExecuteResponseSchema
|
||||
>;
|
||||
@@ -11,6 +11,7 @@ export const CLIENT_CAPS = {
|
||||
// Old clients use a strict TerminalState schema and would reject the extra fields.
|
||||
// Drop the gate (always send the flags) when floor >= v0.1.88.
|
||||
terminalReflowableSnapshot: "terminal_reflowable_snapshot",
|
||||
desktopBrowserAutomation: "desktop_browser_automation",
|
||||
} as const;
|
||||
|
||||
export type ClientCapability = (typeof CLIENT_CAPS)[keyof typeof CLIENT_CAPS];
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { CLIENT_CAPS } from "./client-capabilities.js";
|
||||
import {
|
||||
MutableDaemonConfigPatchSchema,
|
||||
MutableDaemonConfigSchema,
|
||||
SessionInboundMessageSchema,
|
||||
SessionOutboundMessageSchema,
|
||||
WSHelloMessageSchema,
|
||||
} from "./messages.js";
|
||||
|
||||
describe("browser automation protocol integration", () => {
|
||||
test("desktop automation capability parses in hello without narrowing old clients", () => {
|
||||
expect(
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
clientId: "client-1",
|
||||
clientType: "mobile",
|
||||
protocolVersion: 1,
|
||||
capabilities: {
|
||||
[CLIENT_CAPS.desktopBrowserAutomation]: true,
|
||||
},
|
||||
}).capabilities,
|
||||
).toMatchObject({
|
||||
[CLIENT_CAPS.desktopBrowserAutomation]: true,
|
||||
});
|
||||
|
||||
expect(
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
clientId: "old-client",
|
||||
clientType: "mobile",
|
||||
protocolVersion: 1,
|
||||
}).capabilities,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
test("daemon to desktop execute request is an outbound session message", () => {
|
||||
const parsed = SessionOutboundMessageSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
command: { command: "page_info", args: { browserId: "browser-1" } },
|
||||
});
|
||||
|
||||
expect(parsed.type).toBe("browser.automation.execute.request");
|
||||
});
|
||||
|
||||
test("desktop to daemon execute response is an inbound session message", () => {
|
||||
const parsed = SessionInboundMessageSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
},
|
||||
});
|
||||
|
||||
expect(parsed.type).toBe("browser.automation.execute.response");
|
||||
});
|
||||
|
||||
test("mutable daemon config defaults browser tools off and accepts opt-in patches", () => {
|
||||
expect(
|
||||
MutableDaemonConfigSchema.parse({
|
||||
mcp: { injectIntoAgents: false },
|
||||
}).browserTools,
|
||||
).toEqual({ enabled: false });
|
||||
|
||||
expect(
|
||||
MutableDaemonConfigPatchSchema.parse({
|
||||
browserTools: { enabled: true },
|
||||
}).browserTools,
|
||||
).toEqual({ enabled: true });
|
||||
});
|
||||
});
|
||||
@@ -53,6 +53,10 @@ import {
|
||||
LoopLogsResponseSchema,
|
||||
LoopStopResponseSchema,
|
||||
} from "@getpaseo/protocol/loop/rpc-schemas";
|
||||
import {
|
||||
BrowserAutomationExecuteRequestSchema,
|
||||
BrowserAutomationExecuteResponseSchema,
|
||||
} from "./browser-automation/rpc-schemas.js";
|
||||
import {
|
||||
PaseoConfigRawSchema,
|
||||
PaseoLifecycleCommandRawSchema,
|
||||
@@ -129,6 +133,11 @@ export const TerminalProfileSchema = z
|
||||
|
||||
export type TerminalProfile = z.infer<typeof TerminalProfileSchema>;
|
||||
|
||||
const MutableBrowserToolsConfigSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().default(false),
|
||||
})
|
||||
.passthrough();
|
||||
export const MutableDaemonConfigSchema = z
|
||||
.object({
|
||||
mcp: z
|
||||
@@ -136,6 +145,7 @@ export const MutableDaemonConfigSchema = z
|
||||
injectIntoAgents: z.boolean(),
|
||||
})
|
||||
.passthrough(),
|
||||
browserTools: MutableBrowserToolsConfigSchema.default({ enabled: false }),
|
||||
providers: z.record(z.string(), MutableDaemonProviderConfigSchema).default({}),
|
||||
metadataGeneration: MutableMetadataGenerationConfigSchema.default({ providers: [] }),
|
||||
autoArchiveAfterMerge: z.boolean().default(false),
|
||||
@@ -148,6 +158,7 @@ export const MutableDaemonConfigSchema = z
|
||||
export const MutableDaemonConfigPatchSchema = z
|
||||
.object({
|
||||
mcp: MutableDaemonConfigSchema.shape.mcp.partial().optional(),
|
||||
browserTools: MutableBrowserToolsConfigSchema.partial().optional(),
|
||||
providers: z
|
||||
.record(z.string(), MutableDaemonProviderConfigSchema.partial().passthrough())
|
||||
.optional(),
|
||||
@@ -2027,6 +2038,7 @@ export const CaptureTerminalRequestSchema = z.object({
|
||||
});
|
||||
|
||||
export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
|
||||
BrowserAutomationExecuteResponseSchema,
|
||||
VoiceAudioChunkMessageSchema,
|
||||
AbortRequestMessageSchema,
|
||||
AudioPlayedMessageSchema,
|
||||
@@ -4153,6 +4165,7 @@ export const DaemonUpdateProgressMessageSchema = z.object({
|
||||
export type DaemonUpdateProgressMessage = z.infer<typeof DaemonUpdateProgressMessageSchema>;
|
||||
|
||||
export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
||||
BrowserAutomationExecuteRequestSchema,
|
||||
ActivityLogMessageSchema,
|
||||
AssistantChunkMessageSchema,
|
||||
AudioOutputMessageSchema,
|
||||
@@ -4658,6 +4671,7 @@ export const WSHelloMessageSchema = z.object({
|
||||
[CLIENT_CAPS.reasoningMergeEnum]: z.boolean().optional(),
|
||||
[CLIENT_CAPS.customModeIcons]: z.boolean().optional(),
|
||||
[CLIENT_CAPS.terminalReflowableSnapshot]: z.boolean().optional(),
|
||||
[CLIENT_CAPS.desktopBrowserAutomation]: z.boolean().optional(),
|
||||
})
|
||||
.passthrough()
|
||||
.optional(),
|
||||
|
||||
Reference in New Issue
Block a user