Add opt-in browser tools for desktop tabs
* Add opt-in browser tools for desktop tabs Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs. * Fix browser tools CI expectations * Address browser tools review findings * Restrict browser file automation paths * Fix browser upload test on Windows * Harden browser navigation inputs * Make browser tools create usable tabs * Update browser MCP empty-state test * Fail browser tab creation when registration times out * Fix browser screenshots for agents * Hide disabled browser tools from agents * Address browser tools architecture review * Replace browser tools review tests * Wrap browser tab registration errors * Mock Expo Router in app unit tests * Handle invalid browser automation requests * Return browser failure on desktop disconnect * Update browser disconnect websocket test * Relax browser timeout polling test * Handle invalid browser responses * Return browser failure when send fails * Remove local diagnostics and fixture paths * Fix dev service home fallback * Use worktree home for dev services * Use managed daemon in desktop dev * fix(browser): keep agent tabs addressable Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution. * refactor(browser): register tools through catalog Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter. * fix(settings): translate browser tools host error
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,343 @@
|
||||
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
|
||||
import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident";
|
||||
import { createWorkspaceBrowser } from "@/stores/browser-store";
|
||||
import {
|
||||
buildWorkspaceTabPersistenceKey,
|
||||
useWorkspaceLayoutStore,
|
||||
} from "@/stores/workspace-layout-store";
|
||||
|
||||
type BrowserAutomationExecuteRequest = Extract<
|
||||
SessionOutboundMessage,
|
||||
{ type: "browser.automation.execute.request" }
|
||||
>;
|
||||
type BrowserAutomationExecuteResponse = Extract<
|
||||
SessionInboundMessage,
|
||||
{ type: "browser.automation.execute.response" }
|
||||
>;
|
||||
type BrowserAutomationResponsePayload = BrowserAutomationExecuteResponse["payload"];
|
||||
type BrowserAutomationFailurePayload = Extract<BrowserAutomationResponsePayload, { ok: false }>;
|
||||
type BrowserAutomationErrorCode = BrowserAutomationFailurePayload["error"]["code"];
|
||||
|
||||
interface BrowserAutomationClient {
|
||||
on(
|
||||
type: "browser.automation.execute.request",
|
||||
handler: (message: BrowserAutomationExecuteRequest) => void,
|
||||
): () => void;
|
||||
sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void;
|
||||
}
|
||||
|
||||
export interface BrowserAutomationHandlerOptions {
|
||||
client: BrowserAutomationClient;
|
||||
serverId?: string;
|
||||
getHost?: () => DesktopHostBridge | null;
|
||||
ensureResidentBrowserWebview?: typeof ensureResidentBrowserWebviewDefault;
|
||||
registrationWaitTimeoutMs?: number;
|
||||
registrationPollIntervalMs?: number;
|
||||
}
|
||||
|
||||
export function mountBrowserAutomationHandler(
|
||||
options: BrowserAutomationHandlerOptions,
|
||||
): () => void {
|
||||
const getHost = options.getHost ?? getDesktopHost;
|
||||
return options.client.on("browser.automation.execute.request", (request) => {
|
||||
void handleBrowserAutomationRequest({
|
||||
client: options.client,
|
||||
getHost,
|
||||
request,
|
||||
serverId: options.serverId,
|
||||
ensureResidentBrowserWebview:
|
||||
options.ensureResidentBrowserWebview ?? ensureResidentBrowserWebviewDefault,
|
||||
...(options.registrationWaitTimeoutMs !== undefined
|
||||
? { registrationWaitTimeoutMs: options.registrationWaitTimeoutMs }
|
||||
: {}),
|
||||
...(options.registrationPollIntervalMs !== undefined
|
||||
? { registrationPollIntervalMs: options.registrationPollIntervalMs }
|
||||
: {}),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function mountBrowserAutomationDaemonClientHandler(
|
||||
client: unknown,
|
||||
options?: { serverId?: string },
|
||||
): () => void {
|
||||
return mountBrowserAutomationHandler({
|
||||
client: client as BrowserAutomationClient,
|
||||
...(options?.serverId ? { serverId: options.serverId } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function handleBrowserAutomationRequest(params: {
|
||||
client: BrowserAutomationHandlerOptions["client"];
|
||||
getHost: () => DesktopHostBridge | null;
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
serverId?: string;
|
||||
ensureResidentBrowserWebview: typeof ensureResidentBrowserWebviewDefault;
|
||||
registrationWaitTimeoutMs?: number;
|
||||
registrationPollIntervalMs?: number;
|
||||
}): Promise<void> {
|
||||
const {
|
||||
client,
|
||||
getHost,
|
||||
request,
|
||||
serverId,
|
||||
ensureResidentBrowserWebview,
|
||||
registrationWaitTimeoutMs,
|
||||
registrationPollIntervalMs,
|
||||
} = params;
|
||||
const browserHost = getHost()?.browser;
|
||||
const executeAutomationCommand = browserHost?.executeAutomationCommand;
|
||||
|
||||
if (request.command.command === "new_tab") {
|
||||
try {
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: await openBrowserTabForRequest({
|
||||
request,
|
||||
serverId,
|
||||
browserHost,
|
||||
ensureResidentBrowserWebview,
|
||||
...(registrationWaitTimeoutMs !== undefined ? { registrationWaitTimeoutMs } : {}),
|
||||
...(registrationPollIntervalMs !== undefined ? { registrationPollIntervalMs } : {}),
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: normalizeThrownBridgeError(request.requestId, error),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
await rememberAgentBrowserTarget({ request, browserHost });
|
||||
|
||||
if (!executeAutomationCommand) {
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_unsupported",
|
||||
message: "Desktop browser automation is not available in this app runtime.",
|
||||
}),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const payload = await executeAutomationCommand(request);
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: normalizeBridgePayload(request.requestId, payload),
|
||||
});
|
||||
} catch (error) {
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: normalizeThrownBridgeError(request.requestId, error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function openBrowserTabForRequest(params: {
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
serverId?: string;
|
||||
browserHost: DesktopHostBridge["browser"] | undefined;
|
||||
ensureResidentBrowserWebview: typeof ensureResidentBrowserWebviewDefault;
|
||||
registrationWaitTimeoutMs?: number;
|
||||
registrationPollIntervalMs?: number;
|
||||
}): Promise<BrowserAutomationResponsePayload> {
|
||||
const {
|
||||
request,
|
||||
serverId,
|
||||
browserHost,
|
||||
ensureResidentBrowserWebview,
|
||||
registrationWaitTimeoutMs,
|
||||
registrationPollIntervalMs,
|
||||
} = params;
|
||||
const command = request.command as Extract<
|
||||
BrowserAutomationExecuteRequest["command"],
|
||||
{ command: "new_tab" }
|
||||
>;
|
||||
const workspaceId = request.workspaceId ?? command.args.workspaceId;
|
||||
if (!serverId || !workspaceId) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_no_tab",
|
||||
message: "Cannot create a browser tab without a workspace context.",
|
||||
});
|
||||
}
|
||||
|
||||
const url = command.args.url ?? "https://example.com";
|
||||
const { browserId, url: normalizedUrl } = createWorkspaceBrowser({ initialUrl: url });
|
||||
const workspaceKey = buildWorkspaceTabPersistenceKey({ serverId, workspaceId });
|
||||
if (!workspaceKey) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_no_tab",
|
||||
message: "Cannot create a browser tab without a workspace context.",
|
||||
});
|
||||
}
|
||||
useWorkspaceLayoutStore.getState().openTabInBackground(workspaceKey, {
|
||||
kind: "browser",
|
||||
browserId,
|
||||
});
|
||||
|
||||
await browserHost?.registerWorkspaceBrowser?.({ browserId, workspaceId });
|
||||
if (request.agentId) {
|
||||
await browserHost?.setAgentActiveBrowser?.({ agentId: request.agentId, browserId });
|
||||
}
|
||||
|
||||
if (browserHost?.executeAutomationCommand) {
|
||||
ensureResidentBrowserWebview({ browserId, url: normalizedUrl });
|
||||
const registered = await waitForBrowserRegistration({
|
||||
request,
|
||||
browserId,
|
||||
workspaceId,
|
||||
executeAutomationCommand: browserHost.executeAutomationCommand,
|
||||
...(registrationWaitTimeoutMs !== undefined ? { timeoutMs: registrationWaitTimeoutMs } : {}),
|
||||
...(registrationPollIntervalMs !== undefined
|
||||
? { pollIntervalMs: registrationPollIntervalMs }
|
||||
: {}),
|
||||
});
|
||||
if (!registered) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_timeout",
|
||||
message: `Timed out waiting for browser tab ${browserId} to register with desktop automation. Try browser_new_tab again.`,
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
result: { command: "new_tab", browserId, workspaceId, url: normalizedUrl },
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForBrowserRegistration(params: {
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
browserId: string;
|
||||
workspaceId: string;
|
||||
executeAutomationCommand: (
|
||||
request: BrowserAutomationExecuteRequest,
|
||||
) => Promise<BrowserAutomationResponsePayload>;
|
||||
timeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
}): Promise<boolean> {
|
||||
const deadline = Date.now() + (params.timeoutMs ?? 5_000);
|
||||
while (Date.now() < deadline) {
|
||||
const payload = await params.executeAutomationCommand({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: `${params.request.requestId}:list_tabs`,
|
||||
agentId: params.request.agentId,
|
||||
cwd: params.request.cwd,
|
||||
workspaceId: params.workspaceId,
|
||||
command: { command: "list_tabs", args: { workspaceId: params.workspaceId } },
|
||||
});
|
||||
if (payload.ok && payload.result.command === "list_tabs") {
|
||||
if (payload.result.tabs.some((tab) => tab.browserId === params.browserId)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
await delay(params.pollIntervalMs ?? 100);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function rememberAgentBrowserTarget(input: {
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
browserHost: DesktopHostBridge["browser"] | undefined;
|
||||
}): Promise<void> {
|
||||
if (!input.request.agentId) {
|
||||
return;
|
||||
}
|
||||
const browserId = readRequestBrowserId(input.request);
|
||||
if (!browserId) {
|
||||
return;
|
||||
}
|
||||
await input.browserHost?.setAgentActiveBrowser?.({ agentId: input.request.agentId, browserId });
|
||||
}
|
||||
|
||||
function readRequestBrowserId(request: BrowserAutomationExecuteRequest): string | null {
|
||||
if (request.browserId) {
|
||||
return request.browserId;
|
||||
}
|
||||
const args = request.command.args as { browserId?: unknown };
|
||||
return typeof args.browserId === "string" && args.browserId.length > 0 ? args.browserId : null;
|
||||
}
|
||||
|
||||
function normalizeBridgePayload(
|
||||
requestId: string,
|
||||
payload: BrowserAutomationResponsePayload,
|
||||
): BrowserAutomationResponsePayload {
|
||||
return { ...payload, requestId } as BrowserAutomationResponsePayload;
|
||||
}
|
||||
|
||||
function normalizeThrownBridgeError(
|
||||
requestId: string,
|
||||
error: unknown,
|
||||
): BrowserAutomationFailurePayload {
|
||||
const typed = readTypedBrowserAutomationError(error);
|
||||
if (typed) {
|
||||
return browserAutomationFailure({ requestId, ...typed });
|
||||
}
|
||||
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (message.includes("No handler registered")) {
|
||||
return browserAutomationFailure({
|
||||
requestId,
|
||||
code: "browser_unsupported",
|
||||
message: "Desktop browser automation is not implemented by this desktop build yet.",
|
||||
});
|
||||
}
|
||||
|
||||
return browserAutomationFailure({
|
||||
requestId,
|
||||
code: "browser_unknown_error",
|
||||
message: message || "Desktop browser automation failed.",
|
||||
});
|
||||
}
|
||||
|
||||
function readTypedBrowserAutomationError(
|
||||
value: unknown,
|
||||
): { code: BrowserAutomationErrorCode; message: string; retryable?: boolean } | null {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
if (typeof record.code !== "string" || !record.code.startsWith("browser_")) {
|
||||
return null;
|
||||
}
|
||||
if (typeof record.message !== "string" || record.message.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
code: record.code as BrowserAutomationErrorCode,
|
||||
message: record.message,
|
||||
...(typeof record.retryable === "boolean" ? { retryable: record.retryable } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function browserAutomationFailure(params: {
|
||||
requestId: string;
|
||||
code: BrowserAutomationErrorCode;
|
||||
message: string;
|
||||
retryable?: boolean;
|
||||
}): BrowserAutomationFailurePayload {
|
||||
return {
|
||||
requestId: params.requestId,
|
||||
ok: false,
|
||||
error: {
|
||||
code: params.code,
|
||||
message: params.message,
|
||||
retryable: params.retryable ?? false,
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user