Add opt-in browser tools for desktop tabs

* Add opt-in browser tools for desktop tabs

Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs.

* Fix browser tools CI expectations

* Address browser tools review findings

* Restrict browser file automation paths

* Fix browser upload test on Windows

* Harden browser navigation inputs

* Make browser tools create usable tabs

* Update browser MCP empty-state test

* Fail browser tab creation when registration times out

* Fix browser screenshots for agents

* Hide disabled browser tools from agents

* Address browser tools architecture review

* Replace browser tools review tests

* Wrap browser tab registration errors

* Mock Expo Router in app unit tests

* Handle invalid browser automation requests

* Return browser failure on desktop disconnect

* Update browser disconnect websocket test

* Relax browser timeout polling test

* Handle invalid browser responses

* Return browser failure when send fails

* Remove local diagnostics and fixture paths

* Fix dev service home fallback

* Use worktree home for dev services

* Use managed daemon in desktop dev

* fix(browser): keep agent tabs addressable

Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution.

* refactor(browser): register tools through catalog

Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter.

* fix(settings): translate browser tools host error
This commit is contained in:
Mohamed Boudra
2026-06-30 22:33:18 +02:00
committed by GitHub
parent db03b1f3fd
commit 20385bdb50
71 changed files with 10908 additions and 115 deletions
@@ -0,0 +1,599 @@
import { z } from "zod";
export const BrowserAutomationErrorCodeSchema = z.enum([
"browser_disabled",
"browser_no_desktop",
"browser_no_tab",
"browser_tab_not_found",
"browser_tab_closed",
"browser_timeout",
"browser_denied",
"browser_unsupported",
"browser_stale_ref",
"browser_unknown_error",
]);
const BrowserAutomationTabTargetSchema = z.object({
workspaceId: z.string().min(1).optional(),
browserId: z.string().min(1).optional(),
});
const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/);
const BrowserAutomationHttpUrlSchema = z
.string()
.url()
.refine((value) => {
const protocol = new URL(value).protocol;
return protocol === "http:" || protocol === "https:";
}, "URL must use http or https");
export const BrowserAutomationListTabsCommandSchema = z.object({
command: z.literal("list_tabs"),
args: z
.object({
workspaceId: z.string().min(1).optional(),
})
.default({}),
});
export const BrowserAutomationNewTabCommandSchema = z.object({
command: z.literal("new_tab"),
args: z
.object({
workspaceId: z.string().min(1).optional(),
url: BrowserAutomationHttpUrlSchema.optional(),
})
.default({}),
});
export const BrowserAutomationPageInfoCommandSchema = z.object({
command: z.literal("page_info"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationSnapshotCommandSchema = z.object({
command: z.literal("snapshot"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationClickCommandSchema = z.object({
command: z.literal("click"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationFillCommandSchema = z.object({
command: z.literal("fill"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
value: z.string(),
}),
});
export const BrowserAutomationWaitCommandSchema = z.object({
command: z.literal("wait"),
args: BrowserAutomationTabTargetSchema.extend({
text: z.string().min(1).optional(),
url: z.string().min(1).optional(),
timeoutMs: z.number().int().positive().max(30_000).optional(),
}),
});
export const BrowserAutomationTypeCommandSchema = z.object({
command: z.literal("type"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema.optional(),
text: z.string(),
}),
});
export const BrowserAutomationKeypressCommandSchema = z.object({
command: z.literal("keypress"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema.optional(),
key: z.string().min(1),
}),
});
export const BrowserAutomationNavigateCommandSchema = z.object({
command: z.literal("navigate"),
args: BrowserAutomationTabTargetSchema.extend({
url: BrowserAutomationHttpUrlSchema,
}),
});
export const BrowserAutomationBackCommandSchema = z.object({
command: z.literal("back"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationForwardCommandSchema = z.object({
command: z.literal("forward"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationReloadCommandSchema = z.object({
command: z.literal("reload"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationScreenshotCommandSchema = z.object({
command: z.literal("screenshot"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({
command: z.literal("full_page_screenshot"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationPdfCommandSchema = z.object({
command: z.literal("pdf"),
args: BrowserAutomationTabTargetSchema.extend({
landscape: z.boolean().optional(),
printBackground: z.boolean().default(true),
}).default({ printBackground: true }),
});
export const BrowserAutomationDownloadCommandSchema = z.object({
command: z.literal("download"),
args: BrowserAutomationTabTargetSchema.extend({
url: BrowserAutomationHttpUrlSchema,
fileName: z.string().min(1).optional(),
}),
});
export const BrowserAutomationUploadCommandSchema = z.object({
command: z.literal("upload"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
filePaths: z.array(z.string().min(1)).min(1),
}),
});
export const BrowserAutomationFocusCommandSchema = z.object({
command: z.literal("focus"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationClearCommandSchema = z.object({
command: z.literal("clear"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationCheckCommandSchema = z.object({
command: z.literal("check"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
checked: z.boolean().default(true),
}),
});
export const BrowserAutomationSelectCommandSchema = z.object({
command: z.literal("select"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
value: z.string(),
}),
});
export const BrowserAutomationHoverCommandSchema = z.object({
command: z.literal("hover"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationDragCommandSchema = z.object({
command: z.literal("drag"),
args: BrowserAutomationTabTargetSchema.extend({
sourceRef: BrowserAutomationRefSchema,
targetRef: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationLogsCommandSchema = z.object({
command: z.literal("logs"),
args: BrowserAutomationTabTargetSchema.extend({
maxEntries: z.number().int().positive().max(200).default(50),
}).default({ maxEntries: 50 }),
});
export const BrowserAutomationStorageCommandSchema = z.object({
command: z.literal("storage"),
args: BrowserAutomationTabTargetSchema.default({}),
});
const BrowserAutomationViewportInputSchema = z.object({
width: z.number().int().positive(),
height: z.number().int().positive(),
deviceScaleFactor: z.number().positive().optional(),
});
const BrowserAutomationGeolocationInputSchema = z.object({
latitude: z.number().min(-90).max(90),
longitude: z.number().min(-180).max(180),
accuracy: z.number().positive().optional(),
});
export const BrowserAutomationEnvironmentCommandSchema = z.object({
command: z.literal("environment"),
args: BrowserAutomationTabTargetSchema.extend({
viewport: BrowserAutomationViewportInputSchema.optional(),
geolocation: BrowserAutomationGeolocationInputSchema.optional(),
}).default({}),
});
export const BrowserAutomationSetBackgroundCommandSchema = z.object({
command: z.literal("set_background"),
args: BrowserAutomationTabTargetSchema.extend({
color: z.string().min(1),
}),
});
export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsCommandSchema,
BrowserAutomationNewTabCommandSchema,
BrowserAutomationPageInfoCommandSchema,
BrowserAutomationSnapshotCommandSchema,
BrowserAutomationClickCommandSchema,
BrowserAutomationFillCommandSchema,
BrowserAutomationWaitCommandSchema,
BrowserAutomationTypeCommandSchema,
BrowserAutomationKeypressCommandSchema,
BrowserAutomationNavigateCommandSchema,
BrowserAutomationBackCommandSchema,
BrowserAutomationForwardCommandSchema,
BrowserAutomationReloadCommandSchema,
BrowserAutomationScreenshotCommandSchema,
BrowserAutomationFullPageScreenshotCommandSchema,
BrowserAutomationPdfCommandSchema,
BrowserAutomationDownloadCommandSchema,
BrowserAutomationUploadCommandSchema,
BrowserAutomationFocusCommandSchema,
BrowserAutomationClearCommandSchema,
BrowserAutomationCheckCommandSchema,
BrowserAutomationSelectCommandSchema,
BrowserAutomationHoverCommandSchema,
BrowserAutomationDragCommandSchema,
BrowserAutomationLogsCommandSchema,
BrowserAutomationStorageCommandSchema,
BrowserAutomationEnvironmentCommandSchema,
BrowserAutomationSetBackgroundCommandSchema,
]);
export const BrowserAutomationTabInfoSchema = z.object({
browserId: z.string().min(1),
workspaceId: z.string().min(1).optional(),
url: z.string(),
title: z.string(),
isActive: z.boolean().default(false),
isLoading: z.boolean().default(false),
canGoBack: z.boolean().optional(),
canGoForward: z.boolean().optional(),
});
export const BrowserAutomationListTabsResultSchema = z.object({
command: z.literal("list_tabs"),
tabs: z.array(BrowserAutomationTabInfoSchema),
});
export const BrowserAutomationNewTabResultSchema = z.object({
command: z.literal("new_tab"),
browserId: z.string().min(1),
workspaceId: z.string().min(1),
url: z.string().min(1),
});
export const BrowserAutomationPageInfoResultSchema = z.object({
command: z.literal("page_info"),
tab: BrowserAutomationTabInfoSchema,
});
export const BrowserAutomationSnapshotElementSchema = z.object({
ref: z.string().regex(/^@e\d+$/),
role: z.string(),
tagName: z.string(),
text: z.string(),
selector: z.string(),
attributes: z.record(z.string(), z.string()).default({}),
});
export const BrowserAutomationSnapshotResultSchema = z.object({
command: z.literal("snapshot"),
browserId: z.string().min(1),
workspaceId: z.string().min(1).optional(),
url: z.string(),
title: z.string(),
elements: z.array(BrowserAutomationSnapshotElementSchema),
});
export const BrowserAutomationClickResultSchema = z.object({
command: z.literal("click"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationFillResultSchema = z.object({
command: z.literal("fill"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationWaitResultSchema = z.object({
command: z.literal("wait"),
browserId: z.string().min(1),
matched: z.enum(["text", "url"]),
});
export const BrowserAutomationTypeResultSchema = z.object({
command: z.literal("type"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
});
export const BrowserAutomationKeypressResultSchema = z.object({
command: z.literal("keypress"),
browserId: z.string().min(1),
key: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
});
export const BrowserAutomationNavigateResultSchema = z.object({
command: z.literal("navigate"),
browserId: z.string().min(1),
url: z.string().min(1),
});
export const BrowserAutomationBackResultSchema = z.object({
command: z.literal("back"),
browserId: z.string().min(1),
});
export const BrowserAutomationForwardResultSchema = z.object({
command: z.literal("forward"),
browserId: z.string().min(1),
});
export const BrowserAutomationReloadResultSchema = z.object({
command: z.literal("reload"),
browserId: z.string().min(1),
});
export const BrowserAutomationScreenshotResultSchema = z.object({
command: z.literal("screenshot"),
browserId: z.string().min(1),
mimeType: z.literal("image/png"),
dataBase64: z.string().min(1),
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
});
export const BrowserAutomationFullPageScreenshotResultSchema = z.object({
command: z.literal("full_page_screenshot"),
browserId: z.string().min(1),
mimeType: z.literal("image/png"),
dataBase64: z.string().min(1),
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
});
export const BrowserAutomationPdfResultSchema = z.object({
command: z.literal("pdf"),
browserId: z.string().min(1),
mimeType: z.literal("application/pdf"),
dataBase64: z.string().min(1),
});
export const BrowserAutomationDownloadResultSchema = z.object({
command: z.literal("download"),
browserId: z.string().min(1),
url: z.string().min(1),
filePath: z.string().min(1),
totalBytes: z.number().int().nonnegative().optional(),
state: z.string().min(1),
});
export const BrowserAutomationUploadResultSchema = z.object({
command: z.literal("upload"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
filePaths: z.array(z.string().min(1)).min(1),
});
export const BrowserAutomationFocusResultSchema = z.object({
command: z.literal("focus"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationClearResultSchema = z.object({
command: z.literal("clear"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationCheckResultSchema = z.object({
command: z.literal("check"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
checked: z.boolean(),
});
export const BrowserAutomationSelectResultSchema = z.object({
command: z.literal("select"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
value: z.string(),
});
export const BrowserAutomationHoverResultSchema = z.object({
command: z.literal("hover"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationDragResultSchema = z.object({
command: z.literal("drag"),
browserId: z.string().min(1),
sourceRef: BrowserAutomationRefSchema,
targetRef: BrowserAutomationRefSchema,
});
export const BrowserAutomationConsoleLogEntrySchema = z.object({
level: z.string(),
message: z.string(),
source: z.string().optional(),
line: z.number().int().optional(),
timestamp: z.number(),
});
export const BrowserAutomationNetworkLogEntrySchema = z.object({
url: z.string(),
method: z.string().optional(),
status: z.number().int().optional(),
type: z.string().optional(),
startTime: z.number(),
duration: z.number(),
transferSize: z.number().optional(),
});
export const BrowserAutomationLogsResultSchema = z.object({
command: z.literal("logs"),
browserId: z.string().min(1),
console: z.array(BrowserAutomationConsoleLogEntrySchema),
network: z.array(BrowserAutomationNetworkLogEntrySchema),
});
export const BrowserAutomationCookieEntrySchema = z.object({
name: z.string(),
value: z.string(),
domain: z.string().optional(),
path: z.string().optional(),
secure: z.boolean().optional(),
httpOnly: z.boolean().optional(),
expirationDate: z.number().optional(),
});
export const BrowserAutomationStorageEntrySchema = z.object({
key: z.string(),
value: z.string(),
});
export const BrowserAutomationStorageResultSchema = z.object({
command: z.literal("storage"),
browserId: z.string().min(1),
url: z.string(),
cookies: z.array(BrowserAutomationCookieEntrySchema),
localStorage: z.array(BrowserAutomationStorageEntrySchema),
sessionStorage: z.array(BrowserAutomationStorageEntrySchema),
});
export const BrowserAutomationViewportResultSchema = z.object({
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
deviceScaleFactor: z.number().positive(),
});
export const BrowserAutomationGeolocationResultSchema = z.object({
latitude: z.number(),
longitude: z.number(),
accuracy: z.number(),
});
export const BrowserAutomationEnvironmentResultSchema = z.object({
command: z.literal("environment"),
browserId: z.string().min(1),
viewport: BrowserAutomationViewportResultSchema,
geolocation: BrowserAutomationGeolocationResultSchema.optional(),
});
export const BrowserAutomationSetBackgroundResultSchema = z.object({
command: z.literal("set_background"),
browserId: z.string().min(1),
color: z.string().min(1),
});
export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsResultSchema,
BrowserAutomationNewTabResultSchema,
BrowserAutomationPageInfoResultSchema,
BrowserAutomationSnapshotResultSchema,
BrowserAutomationClickResultSchema,
BrowserAutomationFillResultSchema,
BrowserAutomationWaitResultSchema,
BrowserAutomationTypeResultSchema,
BrowserAutomationKeypressResultSchema,
BrowserAutomationNavigateResultSchema,
BrowserAutomationBackResultSchema,
BrowserAutomationForwardResultSchema,
BrowserAutomationReloadResultSchema,
BrowserAutomationScreenshotResultSchema,
BrowserAutomationFullPageScreenshotResultSchema,
BrowserAutomationPdfResultSchema,
BrowserAutomationDownloadResultSchema,
BrowserAutomationUploadResultSchema,
BrowserAutomationFocusResultSchema,
BrowserAutomationClearResultSchema,
BrowserAutomationCheckResultSchema,
BrowserAutomationSelectResultSchema,
BrowserAutomationHoverResultSchema,
BrowserAutomationDragResultSchema,
BrowserAutomationLogsResultSchema,
BrowserAutomationStorageResultSchema,
BrowserAutomationEnvironmentResultSchema,
BrowserAutomationSetBackgroundResultSchema,
]);
export const BrowserAutomationErrorSchema = z.object({
code: BrowserAutomationErrorCodeSchema,
message: z.string().min(1),
retryable: z.boolean().default(false),
});
export const BrowserAutomationExecuteRequestSchema = z.object({
type: z.literal("browser.automation.execute.request"),
requestId: z.string().min(1),
agentId: z.string().min(1).optional(),
cwd: z.string().min(1).optional(),
workspaceId: z.string().min(1).optional(),
browserId: z.string().min(1).optional(),
command: BrowserAutomationCommandSchema,
});
export const BrowserAutomationExecuteResponseSchema = z.object({
type: z.literal("browser.automation.execute.response"),
payload: z.discriminatedUnion("ok", [
z.object({
requestId: z.string().min(1),
ok: z.literal(true),
result: BrowserAutomationResultSchema,
}),
z.object({
requestId: z.string().min(1),
ok: z.literal(false),
error: BrowserAutomationErrorSchema,
}),
]),
});
export type BrowserAutomationErrorCode = z.infer<typeof BrowserAutomationErrorCodeSchema>;
export type BrowserAutomationCommand = z.infer<typeof BrowserAutomationCommandSchema>;
export type BrowserAutomationResult = z.infer<typeof BrowserAutomationResultSchema>;
export type BrowserAutomationConsoleLogEntry = z.infer<
typeof BrowserAutomationConsoleLogEntrySchema
>;
export type BrowserAutomationNetworkLogEntry = z.infer<
typeof BrowserAutomationNetworkLogEntrySchema
>;
export type BrowserAutomationCookieEntry = z.infer<typeof BrowserAutomationCookieEntrySchema>;
export type BrowserAutomationStorageEntry = z.infer<typeof BrowserAutomationStorageEntrySchema>;
export type BrowserAutomationExecuteRequest = z.infer<typeof BrowserAutomationExecuteRequestSchema>;
export type BrowserAutomationExecuteResponse = z.infer<
typeof BrowserAutomationExecuteResponseSchema
>;