Add opt-in browser tools for desktop tabs

* Add opt-in browser tools for desktop tabs

Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs.

* Fix browser tools CI expectations

* Address browser tools review findings

* Restrict browser file automation paths

* Fix browser upload test on Windows

* Harden browser navigation inputs

* Make browser tools create usable tabs

* Update browser MCP empty-state test

* Fail browser tab creation when registration times out

* Fix browser screenshots for agents

* Hide disabled browser tools from agents

* Address browser tools architecture review

* Replace browser tools review tests

* Wrap browser tab registration errors

* Mock Expo Router in app unit tests

* Handle invalid browser automation requests

* Return browser failure on desktop disconnect

* Update browser disconnect websocket test

* Relax browser timeout polling test

* Handle invalid browser responses

* Return browser failure when send fails

* Remove local diagnostics and fixture paths

* Fix dev service home fallback

* Use worktree home for dev services

* Use managed daemon in desktop dev

* fix(browser): keep agent tabs addressable

Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution.

* refactor(browser): register tools through catalog

Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter.

* fix(settings): translate browser tools host error
This commit is contained in:
Mohamed Boudra
2026-06-30 22:33:18 +02:00
committed by GitHub
parent db03b1f3fd
commit 20385bdb50
71 changed files with 10908 additions and 115 deletions
+1 -1
View File
@@ -138,7 +138,7 @@ Electron wrapper for macOS, Linux, and Windows.
> **Window-state v1 limitation:** only the _first_ window of a session restores and persists saved geometry (size/position/maximized). Windows opened via ⌘⇧N / second-instance / "Open in new window" open at the default size, OS-cascaded, and do not persist — this avoids every window stacking on the same restored bounds and fighting over the single window-state store. Lifting this needs per-window state keys. > **Window-state v1 limitation:** only the _first_ window of a session restores and persists saved geometry (size/position/maximized). Windows opened via ⌘⇧N / second-instance / "Open in new window" open at the default size, OS-cascaded, and do not persist — this avoids every window stacking on the same restored bounds and fighting over the single window-state store. Lifting this needs per-window state keys.
> >
> **In-app browser panes are not yet per-window.** The active-browser id (`features/browser-webviews.ts`) and the webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) are process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up. > **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus and agent automation targets are intentionally separate: the workspace-active browser follows the user's focused tab, while the agent-active browser is the default target for browser MCP commands. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up.
### `packages/website` — Marketing site ### `packages/website` — Marketing site
+4
View File
@@ -72,6 +72,10 @@ Starting the service must not create, focus, reveal, or leave behind macOS Simul
It launches its own Electron-flavored Expo server and passes that URL to Electron. It launches its own Electron-flavored Expo server and passes that URL to Electron.
Override the CDP port with `PASEO_ELECTRON_REMOTE_DEBUGGING_PORT` when `9223` is busy. Override the CDP port with `PASEO_ELECTRON_REMOTE_DEBUGGING_PORT` when `9223` is busy.
When running a dedicated Electron QA instance against a non-default Expo port, set
`EXPO_DEV_URL` explicitly. Desktop main defaults to `http://localhost:8081`, so
`PASEO_PORT=57928` alone starts Metro on 57928 but Electron still loads 8081.
### React render profiling ### React render profiling
The app has a gated React render profiler in The app has a gated React render profiler in
+1 -1
View File
@@ -9,7 +9,7 @@ Replace the OpenCode provider's per-directory `/event` stream with OpenCode's `/
## Environment ## Environment
- `opencode --version`: `1.14.46` - `opencode --version`: `1.14.46`
- `which opencode`: `/Users/moboudra/.asdf/installs/nodejs/22.20.0/bin/opencode` - `which opencode`: `opencode`
- `node --version`: `v22.20.0` - `node --version`: `v22.20.0`
- `npm --version`: `10.9.3` - `npm --version`: `10.9.3`
+4
View File
@@ -87,6 +87,7 @@ import { polyfillCrypto } from "@/polyfills/crypto";
import { queryClient } from "@/query/query-client"; import { queryClient } from "@/query/query-client";
import { import {
getHostRuntimeStore, getHostRuntimeStore,
hasConfiguredLocalDaemonOverride,
useHostRegistryLoaded, useHostRegistryLoaded,
useHostMutations, useHostMutations,
useHostRuntimeClient, useHostRuntimeClient,
@@ -313,6 +314,9 @@ async function shouldStartBuiltInDaemon(): Promise<boolean> {
if (!shouldUseDesktopDaemon()) { if (!shouldUseDesktopDaemon()) {
return false; return false;
} }
if (hasConfiguredLocalDaemonOverride()) {
return false;
}
const settings = await loadDesktopSettings(); const settings = await loadDesktopSettings();
return settings.daemon.manageBuiltInDaemon; return settings.daemon.manageBuiltInDaemon;
} }
@@ -0,0 +1,375 @@
import { beforeEach, describe, expect, test, vi } from "vitest";
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
import { mountBrowserAutomationHandler } from "./handler";
import type { DesktopHostBridge } from "@/desktop/host";
import { useBrowserStore } from "@/stores/browser-store";
import {
buildWorkspaceTabPersistenceKey,
useWorkspaceLayoutStore,
} from "@/stores/workspace-layout-store";
vi.mock("expo-router", () => ({
router: {
navigate: vi.fn(),
},
}));
vi.mock("@react-native-async-storage/async-storage", () => ({
default: {
getItem: vi.fn(async () => null),
setItem: vi.fn(async () => undefined),
removeItem: vi.fn(async () => undefined),
},
}));
type BrowserAutomationExecuteRequest = Extract<
SessionOutboundMessage,
{ type: "browser.automation.execute.request" }
>;
type BrowserAutomationExecuteResponse = Extract<
SessionInboundMessage,
{ type: "browser.automation.execute.response" }
>;
class FakeDaemonClient {
public sentResponses: BrowserAutomationExecuteResponse[] = [];
private handler: ((request: BrowserAutomationExecuteRequest) => void) | null = null;
public on(
type: "browser.automation.execute.request",
handler: (request: BrowserAutomationExecuteRequest) => void,
): () => void {
expect(type).toBe("browser.automation.execute.request");
this.handler = handler;
return () => {
if (this.handler === handler) {
this.handler = null;
}
};
}
public sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void {
this.sentResponses.push(response);
}
public receive(nextRequest: BrowserAutomationExecuteRequest): void {
this.handler?.(nextRequest);
}
}
function browserAutomationRequest(): BrowserAutomationExecuteRequest {
return {
type: "browser.automation.execute.request",
requestId: "req-1",
command: { command: "list_tabs", args: {} },
};
}
function browserNewTabRequest(): BrowserAutomationExecuteRequest {
return {
type: "browser.automation.execute.request",
requestId: "req-new",
agentId: "agent-1",
workspaceId: "/repo",
command: { command: "new_tab", args: { workspaceId: "/repo", url: "https://example.com" } },
};
}
function emptyListTabsPayload(requestId = "req-new:list_tabs") {
return {
requestId,
ok: true as const,
result: {
command: "list_tabs" as const,
tabs: [],
},
};
}
function currentListTabsPayload(requestId = "req-new:list_tabs") {
return {
requestId,
ok: true as const,
result: {
command: "list_tabs" as const,
tabs: currentBrowserTabs(),
},
};
}
function flushPromises(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 0));
}
function waitForAsyncWork(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 20));
}
function currentBrowserTabs() {
return Object.values(useBrowserStore.getState().browsersById).map((browser) => ({
browserId: browser.browserId,
workspaceId: "/repo",
url: browser.url,
title: browser.title,
isActive: true,
isLoading: false,
}));
}
describe("mountBrowserAutomationHandler", () => {
beforeEach(() => {
useBrowserStore.setState({ browsersById: {} });
useWorkspaceLayoutStore.setState({ layoutByWorkspace: {} });
});
test("creates an unfocused workspace browser tab for browser_new_tab", async () => {
const client = new FakeDaemonClient();
const setWorkspaceActiveBrowser = vi.fn(async () => undefined);
const setAgentActiveBrowser = vi.fn(async () => undefined);
const registerWorkspaceBrowser = vi.fn(async () => undefined);
const ensureResidentBrowserWebview = vi.fn();
const executeAutomationCommand = vi.fn(async () => currentListTabsPayload());
const workspaceKey = buildWorkspaceTabPersistenceKey({
serverId: "server-1",
workspaceId: "/repo",
});
if (!workspaceKey) throw new Error("expected workspace key");
const focusedTabId = useWorkspaceLayoutStore
.getState()
.openTabFocused(workspaceKey, { kind: "draft", draftId: "human-draft" });
if (!focusedTabId) throw new Error("expected focused tab");
mountBrowserAutomationHandler({
client,
serverId: "server-1",
getHost: () =>
({
browser: {
executeAutomationCommand,
registerWorkspaceBrowser,
setWorkspaceActiveBrowser,
setAgentActiveBrowser,
},
}) satisfies DesktopHostBridge,
ensureResidentBrowserWebview,
});
client.receive(browserNewTabRequest());
await flushPromises();
const payload = client.sentResponses[0]?.payload;
expect(payload?.ok).toBe(true);
if (!payload?.ok) throw new Error("expected success");
expect(payload.result.command).toBe("new_tab");
if (payload.result.command !== "new_tab") throw new Error("expected new_tab result");
expect(payload.result.url).toBe("https://example.com");
expect(useWorkspaceLayoutStore.getState().getWorkspaceTabs(workspaceKey)).toContainEqual(
expect.objectContaining({ target: { kind: "browser", browserId: payload.result.browserId } }),
);
const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey];
expect(layout?.root.kind).toBe("pane");
if (layout?.root.kind !== "pane") throw new Error("expected root pane");
expect(layout.root.pane.focusedTabId).toBe(focusedTabId);
expect(registerWorkspaceBrowser).toHaveBeenCalledWith({
browserId: payload.result.browserId,
workspaceId: "/repo",
});
expect(setAgentActiveBrowser).toHaveBeenCalledWith({
agentId: "agent-1",
browserId: payload.result.browserId,
});
expect(setWorkspaceActiveBrowser).not.toHaveBeenCalled();
expect(ensureResidentBrowserWebview).toHaveBeenCalledWith({
browserId: payload.result.browserId,
url: "https://example.com",
});
expect(executeAutomationCommand).toHaveBeenCalledTimes(1);
});
test("returns browser_timeout when the resident webview does not register", async () => {
const client = new FakeDaemonClient();
const ensureResidentBrowserWebview = vi.fn();
const executeAutomationCommand = vi.fn(async () => emptyListTabsPayload());
mountBrowserAutomationHandler({
client,
serverId: "server-1",
getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge,
ensureResidentBrowserWebview,
registrationWaitTimeoutMs: 1,
registrationPollIntervalMs: 1,
});
client.receive(browserNewTabRequest());
await waitForAsyncWork();
expect(client.sentResponses[0]?.payload).toMatchObject({
requestId: "req-new",
ok: false,
error: {
code: "browser_timeout",
retryable: true,
},
});
expect(ensureResidentBrowserWebview).toHaveBeenCalledWith(
expect.objectContaining({ url: "https://example.com" }),
);
expect(client.sentResponses[0]?.payload).not.toMatchObject({
ok: true,
result: { command: "new_tab" },
});
});
test("wraps browser_new_tab registration bridge errors in a response", async () => {
const client = new FakeDaemonClient();
const executeAutomationCommand = vi.fn(async () => {
throw new Error("IPC registration check failed");
});
mountBrowserAutomationHandler({
client,
serverId: "server-1",
getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge,
});
client.receive(browserNewTabRequest());
await flushPromises();
expect(client.sentResponses[0]?.payload).toEqual({
requestId: "req-new",
ok: false,
error: {
code: "browser_unknown_error",
message: "IPC registration check failed",
retryable: false,
},
});
});
test("sends a success response from the desktop bridge", async () => {
const client = new FakeDaemonClient();
const executeAutomationCommand = vi.fn(async () => ({
requestId: "req-1",
ok: true as const,
result: { command: "list_tabs" as const, tabs: [] },
}));
mountBrowserAutomationHandler({
client,
getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge,
});
client.receive(browserAutomationRequest());
await flushPromises();
expect(executeAutomationCommand).toHaveBeenCalledWith(browserAutomationRequest());
expect(client.sentResponses).toEqual([
{
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: true,
result: { command: "list_tabs", tabs: [] },
},
},
]);
});
test("missing bridge sends browser_unsupported", async () => {
const client = new FakeDaemonClient();
mountBrowserAutomationHandler({ client, getHost: () => null });
client.receive(browserAutomationRequest());
await flushPromises();
expect(client.sentResponses).toEqual([
{
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: false,
error: {
code: "browser_unsupported",
message: "Desktop browser automation is not available in this app runtime.",
retryable: false,
},
},
},
]);
});
test("typed bridge errors become failure responses", async () => {
const client = new FakeDaemonClient();
mountBrowserAutomationHandler({
client,
getHost: () => ({
browser: {
executeAutomationCommand: async () => {
throw {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
};
},
},
}),
});
client.receive(browserAutomationRequest());
await flushPromises();
expect(client.sentResponses[0]?.payload).toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
},
});
});
test("unimplemented preload IPC reports browser_unsupported", async () => {
const client = new FakeDaemonClient();
mountBrowserAutomationHandler({
client,
getHost: () => ({
browser: {
executeAutomationCommand: async () => {
throw new Error('No handler registered for "paseo:browser:execute-automation-command"');
},
},
}),
});
client.receive(browserAutomationRequest());
await flushPromises();
expect(client.sentResponses[0]?.payload).toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_unsupported",
message: "Desktop browser automation is not implemented by this desktop build yet.",
retryable: false,
},
});
});
test("unsubscribe stops handling requests", async () => {
const client = new FakeDaemonClient();
const executeAutomationCommand = vi.fn(async () => ({
requestId: "req-1",
ok: true as const,
result: { command: "list_tabs" as const, tabs: [] },
}));
const unsubscribe = mountBrowserAutomationHandler({
client,
getHost: () => ({ browser: { executeAutomationCommand } }),
});
unsubscribe();
client.receive(browserAutomationRequest());
await flushPromises();
expect(executeAutomationCommand).not.toHaveBeenCalled();
expect(client.sentResponses).toEqual([]);
});
});
@@ -0,0 +1,343 @@
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident";
import { createWorkspaceBrowser } from "@/stores/browser-store";
import {
buildWorkspaceTabPersistenceKey,
useWorkspaceLayoutStore,
} from "@/stores/workspace-layout-store";
type BrowserAutomationExecuteRequest = Extract<
SessionOutboundMessage,
{ type: "browser.automation.execute.request" }
>;
type BrowserAutomationExecuteResponse = Extract<
SessionInboundMessage,
{ type: "browser.automation.execute.response" }
>;
type BrowserAutomationResponsePayload = BrowserAutomationExecuteResponse["payload"];
type BrowserAutomationFailurePayload = Extract<BrowserAutomationResponsePayload, { ok: false }>;
type BrowserAutomationErrorCode = BrowserAutomationFailurePayload["error"]["code"];
interface BrowserAutomationClient {
on(
type: "browser.automation.execute.request",
handler: (message: BrowserAutomationExecuteRequest) => void,
): () => void;
sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void;
}
export interface BrowserAutomationHandlerOptions {
client: BrowserAutomationClient;
serverId?: string;
getHost?: () => DesktopHostBridge | null;
ensureResidentBrowserWebview?: typeof ensureResidentBrowserWebviewDefault;
registrationWaitTimeoutMs?: number;
registrationPollIntervalMs?: number;
}
export function mountBrowserAutomationHandler(
options: BrowserAutomationHandlerOptions,
): () => void {
const getHost = options.getHost ?? getDesktopHost;
return options.client.on("browser.automation.execute.request", (request) => {
void handleBrowserAutomationRequest({
client: options.client,
getHost,
request,
serverId: options.serverId,
ensureResidentBrowserWebview:
options.ensureResidentBrowserWebview ?? ensureResidentBrowserWebviewDefault,
...(options.registrationWaitTimeoutMs !== undefined
? { registrationWaitTimeoutMs: options.registrationWaitTimeoutMs }
: {}),
...(options.registrationPollIntervalMs !== undefined
? { registrationPollIntervalMs: options.registrationPollIntervalMs }
: {}),
});
});
}
export function mountBrowserAutomationDaemonClientHandler(
client: unknown,
options?: { serverId?: string },
): () => void {
return mountBrowserAutomationHandler({
client: client as BrowserAutomationClient,
...(options?.serverId ? { serverId: options.serverId } : {}),
});
}
async function handleBrowserAutomationRequest(params: {
client: BrowserAutomationHandlerOptions["client"];
getHost: () => DesktopHostBridge | null;
request: BrowserAutomationExecuteRequest;
serverId?: string;
ensureResidentBrowserWebview: typeof ensureResidentBrowserWebviewDefault;
registrationWaitTimeoutMs?: number;
registrationPollIntervalMs?: number;
}): Promise<void> {
const {
client,
getHost,
request,
serverId,
ensureResidentBrowserWebview,
registrationWaitTimeoutMs,
registrationPollIntervalMs,
} = params;
const browserHost = getHost()?.browser;
const executeAutomationCommand = browserHost?.executeAutomationCommand;
if (request.command.command === "new_tab") {
try {
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
payload: await openBrowserTabForRequest({
request,
serverId,
browserHost,
ensureResidentBrowserWebview,
...(registrationWaitTimeoutMs !== undefined ? { registrationWaitTimeoutMs } : {}),
...(registrationPollIntervalMs !== undefined ? { registrationPollIntervalMs } : {}),
}),
});
} catch (error) {
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
payload: normalizeThrownBridgeError(request.requestId, error),
});
}
return;
}
await rememberAgentBrowserTarget({ request, browserHost });
if (!executeAutomationCommand) {
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
payload: browserAutomationFailure({
requestId: request.requestId,
code: "browser_unsupported",
message: "Desktop browser automation is not available in this app runtime.",
}),
});
return;
}
try {
const payload = await executeAutomationCommand(request);
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
payload: normalizeBridgePayload(request.requestId, payload),
});
} catch (error) {
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
payload: normalizeThrownBridgeError(request.requestId, error),
});
}
}
async function openBrowserTabForRequest(params: {
request: BrowserAutomationExecuteRequest;
serverId?: string;
browserHost: DesktopHostBridge["browser"] | undefined;
ensureResidentBrowserWebview: typeof ensureResidentBrowserWebviewDefault;
registrationWaitTimeoutMs?: number;
registrationPollIntervalMs?: number;
}): Promise<BrowserAutomationResponsePayload> {
const {
request,
serverId,
browserHost,
ensureResidentBrowserWebview,
registrationWaitTimeoutMs,
registrationPollIntervalMs,
} = params;
const command = request.command as Extract<
BrowserAutomationExecuteRequest["command"],
{ command: "new_tab" }
>;
const workspaceId = request.workspaceId ?? command.args.workspaceId;
if (!serverId || !workspaceId) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_no_tab",
message: "Cannot create a browser tab without a workspace context.",
});
}
const url = command.args.url ?? "https://example.com";
const { browserId, url: normalizedUrl } = createWorkspaceBrowser({ initialUrl: url });
const workspaceKey = buildWorkspaceTabPersistenceKey({ serverId, workspaceId });
if (!workspaceKey) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_no_tab",
message: "Cannot create a browser tab without a workspace context.",
});
}
useWorkspaceLayoutStore.getState().openTabInBackground(workspaceKey, {
kind: "browser",
browserId,
});
await browserHost?.registerWorkspaceBrowser?.({ browserId, workspaceId });
if (request.agentId) {
await browserHost?.setAgentActiveBrowser?.({ agentId: request.agentId, browserId });
}
if (browserHost?.executeAutomationCommand) {
ensureResidentBrowserWebview({ browserId, url: normalizedUrl });
const registered = await waitForBrowserRegistration({
request,
browserId,
workspaceId,
executeAutomationCommand: browserHost.executeAutomationCommand,
...(registrationWaitTimeoutMs !== undefined ? { timeoutMs: registrationWaitTimeoutMs } : {}),
...(registrationPollIntervalMs !== undefined
? { pollIntervalMs: registrationPollIntervalMs }
: {}),
});
if (!registered) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_timeout",
message: `Timed out waiting for browser tab ${browserId} to register with desktop automation. Try browser_new_tab again.`,
retryable: true,
});
}
}
return {
requestId: request.requestId,
ok: true,
result: { command: "new_tab", browserId, workspaceId, url: normalizedUrl },
};
}
async function waitForBrowserRegistration(params: {
request: BrowserAutomationExecuteRequest;
browserId: string;
workspaceId: string;
executeAutomationCommand: (
request: BrowserAutomationExecuteRequest,
) => Promise<BrowserAutomationResponsePayload>;
timeoutMs?: number;
pollIntervalMs?: number;
}): Promise<boolean> {
const deadline = Date.now() + (params.timeoutMs ?? 5_000);
while (Date.now() < deadline) {
const payload = await params.executeAutomationCommand({
type: "browser.automation.execute.request",
requestId: `${params.request.requestId}:list_tabs`,
agentId: params.request.agentId,
cwd: params.request.cwd,
workspaceId: params.workspaceId,
command: { command: "list_tabs", args: { workspaceId: params.workspaceId } },
});
if (payload.ok && payload.result.command === "list_tabs") {
if (payload.result.tabs.some((tab) => tab.browserId === params.browserId)) {
return true;
}
}
await delay(params.pollIntervalMs ?? 100);
}
return false;
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function rememberAgentBrowserTarget(input: {
request: BrowserAutomationExecuteRequest;
browserHost: DesktopHostBridge["browser"] | undefined;
}): Promise<void> {
if (!input.request.agentId) {
return;
}
const browserId = readRequestBrowserId(input.request);
if (!browserId) {
return;
}
await input.browserHost?.setAgentActiveBrowser?.({ agentId: input.request.agentId, browserId });
}
function readRequestBrowserId(request: BrowserAutomationExecuteRequest): string | null {
if (request.browserId) {
return request.browserId;
}
const args = request.command.args as { browserId?: unknown };
return typeof args.browserId === "string" && args.browserId.length > 0 ? args.browserId : null;
}
function normalizeBridgePayload(
requestId: string,
payload: BrowserAutomationResponsePayload,
): BrowserAutomationResponsePayload {
return { ...payload, requestId } as BrowserAutomationResponsePayload;
}
function normalizeThrownBridgeError(
requestId: string,
error: unknown,
): BrowserAutomationFailurePayload {
const typed = readTypedBrowserAutomationError(error);
if (typed) {
return browserAutomationFailure({ requestId, ...typed });
}
const message = error instanceof Error ? error.message : String(error);
if (message.includes("No handler registered")) {
return browserAutomationFailure({
requestId,
code: "browser_unsupported",
message: "Desktop browser automation is not implemented by this desktop build yet.",
});
}
return browserAutomationFailure({
requestId,
code: "browser_unknown_error",
message: message || "Desktop browser automation failed.",
});
}
function readTypedBrowserAutomationError(
value: unknown,
): { code: BrowserAutomationErrorCode; message: string; retryable?: boolean } | null {
if (typeof value !== "object" || value === null || Array.isArray(value)) {
return null;
}
const record = value as Record<string, unknown>;
if (typeof record.code !== "string" || !record.code.startsWith("browser_")) {
return null;
}
if (typeof record.message !== "string" || record.message.length === 0) {
return null;
}
return {
code: record.code as BrowserAutomationErrorCode,
message: record.message,
...(typeof record.retryable === "boolean" ? { retryable: record.retryable } : {}),
};
}
function browserAutomationFailure(params: {
requestId: string;
code: BrowserAutomationErrorCode;
message: string;
retryable?: boolean;
}): BrowserAutomationFailurePayload {
return {
requestId: params.requestId,
ok: false,
error: {
code: params.code,
message: params.message,
retryable: params.retryable ?? false,
},
};
}
@@ -25,6 +25,11 @@ import {
} from "@/desktop/host"; } from "@/desktop/host";
import { isDev } from "@/constants/platform"; import { isDev } from "@/constants/platform";
import { useBrowserStore, normalizeWorkspaceBrowserUrl } from "@/stores/browser-store"; import { useBrowserStore, normalizeWorkspaceBrowserUrl } from "@/stores/browser-store";
import {
prepareBrowserWebview,
releaseResidentBrowserWebview,
takeResidentBrowserWebview,
} from "./browser-webview-resident";
type ElectronWebview = HTMLElement & { type ElectronWebview = HTMLElement & {
canGoBack?: () => boolean; canGoBack?: () => boolean;
@@ -404,16 +409,16 @@ export function BrowserPane({
initialUrlRef.current, initialUrlRef.current,
browserErrorLabelsRef.current, browserErrorLabelsRef.current,
); );
const webview = document.createElement("webview") as ElectronWebview; const residentWebview = takeResidentBrowserWebview(browserId) as ElectronWebview | null;
const webview = residentWebview ?? (document.createElement("webview") as ElectronWebview);
webviewRef.current = webview; webviewRef.current = webview;
webview.setAttribute("partition", `persist:paseo-browser-${browserId}`); void getDesktopHost()?.browser?.registerWorkspaceBrowser?.({ browserId, workspaceId });
webview.setAttribute("allowpopups", "true"); if (!residentWebview) {
webview.setAttribute("spellcheck", "false"); prepareBrowserWebview(webview, {
webview.setAttribute("autosize", "on"); browserId,
webview.setAttribute( initialUrl: initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current,
"src", });
initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current, }
);
webview.style.display = "flex"; webview.style.display = "flex";
webview.style.flex = "1"; webview.style.flex = "1";
webview.style.width = "100%"; webview.style.width = "100%";
@@ -528,7 +533,14 @@ export function BrowserPane({
webview.removeEventListener("focus", handleWebviewFocus); webview.removeEventListener("focus", handleWebviewFocus);
webview.removeEventListener("mousedown", handleWebviewFocus); webview.removeEventListener("mousedown", handleWebviewFocus);
if (host.contains(webview)) { if (host.contains(webview)) {
host.removeChild(webview); const browserStillExists = Boolean(
useBrowserStore.getState().browsersById[browserIdRef.current],
);
if (browserStillExists) {
releaseResidentBrowserWebview(browserIdRef.current, webview);
} else {
host.removeChild(webview);
}
} }
if (webviewRef.current === webview) { if (webviewRef.current === webview) {
webviewRef.current = null; webviewRef.current = null;
@@ -0,0 +1,60 @@
import { afterEach, describe, expect, it } from "vitest";
import {
clearResidentBrowserWebviewsForTests,
ensureResidentBrowserWebview,
releaseResidentBrowserWebview,
removeResidentBrowserWebview,
takeResidentBrowserWebview,
} from "./browser-webview-resident";
describe("resident browser webviews", () => {
afterEach(() => {
clearResidentBrowserWebviewsForTests();
});
it("keeps a browser webview mounted offscreen and reuses the same node", () => {
const host = document.createElement("div");
const webview = document.createElement("webview");
host.appendChild(webview);
document.body.appendChild(host);
releaseResidentBrowserWebview("browser-a", webview);
expect(host.children).toHaveLength(0);
expect(webview.isConnected).toBe(true);
expect(webview.style.width).toBe("1280px");
expect(webview.style.height).toBe("800px");
const reused = takeResidentBrowserWebview("browser-a");
expect(reused).toBe(webview);
expect(takeResidentBrowserWebview("browser-a")).toBeNull();
});
it("creates a resident webview for an agent-created unfocused tab", () => {
const webview = ensureResidentBrowserWebview({
browserId: "browser-agent",
url: "https://example.com",
});
expect(webview).not.toBeNull();
expect(webview?.isConnected).toBe(true);
expect(webview?.getAttribute("data-paseo-browser-id")).toBe("browser-agent");
expect(webview?.getAttribute("partition")).toBe("persist:paseo-browser-browser-agent");
expect((webview as HTMLUnknownElement & { src?: string })?.src).toContain(
"https://example.com",
);
});
it("removes a resident webview when its browser tab closes", () => {
const webview = ensureResidentBrowserWebview({
browserId: "browser-closed",
url: "https://example.com",
});
removeResidentBrowserWebview("browser-closed");
expect(webview?.isConnected).toBe(false);
expect(takeResidentBrowserWebview("browser-closed")).toBeNull();
});
});
@@ -0,0 +1,153 @@
const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
const RESIDENT_VIEWPORT_WIDTH = 1280;
const RESIDENT_VIEWPORT_HEIGHT = 800;
const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
interface BrowserWebviewElement extends HTMLElement {
src: string;
}
function trimNonEmpty(value: string | null | undefined): string | null {
if (typeof value !== "string") {
return null;
}
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : null;
}
function readDocument(): Document | null {
return typeof document === "undefined" ? null : document;
}
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
if (existing) {
return existing;
}
const host = ownerDocument.createElement("div");
host.id = RESIDENT_BROWSER_HOST_ID;
host.setAttribute("aria-hidden", "true");
host.style.position = "fixed";
host.style.left = "-20000px";
host.style.top = "0";
host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
host.style.overflow = "hidden";
host.style.opacity = "0";
host.style.pointerEvents = "none";
ownerDocument.body.appendChild(host);
return host;
}
function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null {
for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) {
return element as HTMLElement;
}
}
return null;
}
function applyResidentWebviewStyle(webview: HTMLElement): void {
webview.style.display = "block";
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
webview.style.border = "0";
webview.style.background = "transparent";
}
export function prepareBrowserWebview(
webview: HTMLElement,
input: { browserId: string; initialUrl?: string | null },
): void {
webview.setAttribute(BROWSER_ID_ATTRIBUTE, input.browserId);
webview.setAttribute("partition", `persist:paseo-browser-${input.browserId}`);
webview.setAttribute("allowpopups", "true");
webview.setAttribute("spellcheck", "false");
webview.setAttribute("autosize", "on");
if (input.initialUrl) {
(webview as BrowserWebviewElement).src = input.initialUrl;
}
}
export function ensureResidentBrowserWebview(input: {
browserId: string;
url: string;
}): HTMLElement | null {
const browserId = trimNonEmpty(input.browserId);
if (!browserId) {
return null;
}
const ownerDocument = readDocument();
if (!ownerDocument) {
return null;
}
const resident = residentWebviewsByBrowserId.get(browserId) ?? null;
if (resident?.isConnected) {
return resident;
}
const existing = findBrowserWebview(browserId, ownerDocument);
if (existing) {
return existing;
}
const webview = ownerDocument.createElement("webview") as BrowserWebviewElement;
prepareBrowserWebview(webview, { browserId, initialUrl: input.url });
releaseResidentBrowserWebview(browserId, webview);
return webview;
}
export function takeResidentBrowserWebview(browserId: string): HTMLElement | null {
const normalizedBrowserId = trimNonEmpty(browserId);
if (!normalizedBrowserId) {
return null;
}
const webview = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
if (!webview) {
return null;
}
residentWebviewsByBrowserId.delete(normalizedBrowserId);
return webview;
}
export function releaseResidentBrowserWebview(browserId: string, webview: HTMLElement): void {
const normalizedBrowserId = trimNonEmpty(browserId);
if (!normalizedBrowserId) {
webview.remove();
return;
}
const ownerDocument = readDocument();
if (!ownerDocument) {
return;
}
residentWebviewsByBrowserId.set(normalizedBrowserId, webview);
applyResidentWebviewStyle(webview);
getResidentBrowserHost(ownerDocument).appendChild(webview);
}
export function removeResidentBrowserWebview(browserId: string): void {
const normalizedBrowserId = trimNonEmpty(browserId);
if (!normalizedBrowserId) {
return;
}
const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
residentWebviewsByBrowserId.delete(normalizedBrowserId);
resident?.remove();
}
export function clearResidentBrowserWebviewsForTests(): void {
for (const webview of residentWebviewsByBrowserId.values()) {
webview.remove();
}
residentWebviewsByBrowserId.clear();
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
}
+19 -1
View File
@@ -1,5 +1,15 @@
import { Platform } from "react-native"; import { Platform } from "react-native";
import { getElectronHost } from "@/desktop/electron/host"; import { getElectronHost } from "@/desktop/electron/host";
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
type BrowserAutomationExecuteRequest = Extract<
SessionOutboundMessage,
{ type: "browser.automation.execute.request" }
>;
type BrowserAutomationExecuteResponse = Extract<
SessionInboundMessage,
{ type: "browser.automation.execute.response" }
>;
export type DesktopNotificationPermission = "granted" | "denied" | "default"; export type DesktopNotificationPermission = "granted" | "denied" | "default";
@@ -117,9 +127,17 @@ export interface DesktopBrowserNewTabRequestEvent {
} }
export interface DesktopBrowserBridge { export interface DesktopBrowserBridge {
setWorkspaceActiveBrowser?: (browserId: string | null) => Promise<void>; registerWorkspaceBrowser?: (input: { browserId: string; workspaceId: string }) => Promise<void>;
setWorkspaceActiveBrowser?: (input: {
workspaceId: string;
browserId: string | null;
}) => Promise<void>;
setAgentActiveBrowser?: (input: { agentId: string; browserId: string | null }) => Promise<void>;
openDevTools?: (browserId: string) => Promise<unknown>; openDevTools?: (browserId: string) => Promise<unknown>;
clearPartition?: (browserId: string) => Promise<void>; clearPartition?: (browserId: string) => Promise<void>;
executeAutomationCommand?: (
request: BrowserAutomationExecuteRequest,
) => Promise<BrowserAutomationExecuteResponse["payload"]>;
} }
export interface DesktopInvokeBridge { export interface DesktopInvokeBridge {
+56 -3
View File
@@ -1,4 +1,8 @@
import { afterEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
vi.hoisted(() => {
Object.defineProperty(globalThis, "__DEV__", { value: false, configurable: true });
});
import type { import type {
DaemonClient, DaemonClient,
ConnectionState, ConnectionState,
@@ -16,6 +20,10 @@ import {
type HostRuntimeStorage, type HostRuntimeStorage,
} from "./host-runtime"; } from "./host-runtime";
vi.mock("@/browser-automation/handler", () => ({
mountBrowserAutomationDaemonClientHandler: vi.fn(() => () => undefined),
}));
class FakeDaemonClient { class FakeDaemonClient {
private state: ConnectionState = { status: "idle" }; private state: ConnectionState = { status: "idle" };
private listeners = new Set<(status: ConnectionState) => void>(); private listeners = new Set<(status: ConnectionState) => void>();
@@ -342,6 +350,18 @@ function onceHostListMatches(store: HostRuntimeStore, predicate: () => boolean):
}); });
} }
class BrowserClientLifecycle {
public active: Array<{ serverId: string; connectionId: string }> = [];
mount(input: { host: HostProfile; connection: HostConnection }): () => void {
const entry = { serverId: input.host.serverId, connectionId: input.connection.id };
this.active.push(entry);
return () => {
this.active = this.active.filter((current) => current !== entry);
};
}
}
describe("HostRuntimeController", () => { describe("HostRuntimeController", () => {
it("replaces the active relay client when re-pairing changes the daemon public key", async () => { it("replaces the active relay client when re-pairing changes the daemon public key", async () => {
const oldRelay: HostConnection = { const oldRelay: HostConnection = {
@@ -458,6 +478,39 @@ describe("HostRuntimeController", () => {
expect(controller.getSnapshot().connectionStatus).toBe("online"); expect(controller.getSnapshot().connectionStatus).toBe("online");
}); });
it("keeps browser client lifecycle tied to the active host runtime client", async () => {
const host = makeHost({ preferredConnectionId: "direct:lan:6767" });
const fakeClient = makeConnectedProbeClient(12);
const lifecycle = new BrowserClientLifecycle();
const controller = new HostRuntimeController({
host,
deps: {
createClient: () => new FakeDaemonClient() as unknown as DaemonClient,
connectToDaemon: async ({ host: hostProfile, connection }) => ({
client: makeConnectedProbeClient(10) as unknown as DaemonClient,
serverId: hostProfile.serverId,
hostname: connection.id,
}),
getClientId: async () => "cid_runtime_stable",
mountClientHandlers: (input) => lifecycle.mount(input),
},
});
await controller.start({
autoProbe: false,
initialConnection: {
connectionId: "direct:lan:6767",
existingClient: fakeClient as unknown as DaemonClient,
},
});
expect(lifecycle.active).toEqual([{ serverId: "srv_test", connectionId: "direct:lan:6767" }]);
await controller.stop();
expect(lifecycle.active).toEqual([]);
});
it("adopts the first successful probe on startup", async () => { it("adopts the first successful probe on startup", async () => {
const host = makeHost({ preferredConnectionId: "direct:lan:6767" }); const host = makeHost({ preferredConnectionId: "direct:lan:6767" });
const clients: FakeDaemonClient[] = []; const clients: FakeDaemonClient[] = [];
@@ -1431,7 +1484,7 @@ describe("HostRuntimeStore", () => {
entries: [ entries: [
makeFetchAgentsEntry({ makeFetchAgentsEntry({
id: "agent-recent", id: "agent-recent",
cwd: "/Users/moboudra/dev/paseo", cwd: "/workspaces/paseo",
updatedAt: "2026-03-04T12:00:00.000Z", updatedAt: "2026-03-04T12:00:00.000Z",
title: "Recent agent", title: "Recent agent",
}), }),
@@ -1444,7 +1497,7 @@ describe("HostRuntimeStore", () => {
entries: [ entries: [
makeFetchAgentsEntry({ makeFetchAgentsEntry({
id: "agent-stale-attention", id: "agent-stale-attention",
cwd: "/Users/moboudra/dev/paseo-pr67-review", cwd: "/workspaces/paseo-pr67-review",
updatedAt: "2026-02-20T08:00:00.000Z", updatedAt: "2026-02-20T08:00:00.000Z",
title: "Needs triage", title: "Needs triage",
requiresAttention: true, requiresAttention: true,
@@ -1612,7 +1665,7 @@ describe("HostRuntimeStore", () => {
useSessionStore.getState().setAgents(host.serverId, () => { useSessionStore.getState().setAgents(host.serverId, () => {
const stale = makeFetchAgentsEntry({ const stale = makeFetchAgentsEntry({
id: "agent-archived", id: "agent-archived",
cwd: "/Users/moboudra/dev/paseo", cwd: "/workspaces/paseo",
updatedAt: "2026-03-30T15:29:00.000Z", updatedAt: "2026-03-30T15:29:00.000Z",
archivedAt: null, archivedAt: null,
title: "Stale active copy", title: "Stale active copy",
+33
View File
@@ -37,6 +37,8 @@ import {
buildLocalDaemonTransportUrl, buildLocalDaemonTransportUrl,
createDesktopLocalDaemonTransportFactory, createDesktopLocalDaemonTransportFactory,
} from "@/desktop/daemon/desktop-daemon-transport"; } from "@/desktop/daemon/desktop-daemon-transport";
import { getDesktopHost } from "@/desktop/host";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
import { replaceFetchedAgentDirectory } from "@/utils/agent-directory-sync"; import { replaceFetchedAgentDirectory } from "@/utils/agent-directory-sync";
import { useSessionStore } from "@/stores/session-store"; import { useSessionStore } from "@/stores/session-store";
import { import {
@@ -45,6 +47,7 @@ import {
} from "@/workspace/legacy-daemon-workspaces"; } from "@/workspace/legacy-daemon-workspaces";
import { invalidateCheckoutGitQueriesForServer } from "@/git/query-keys"; import { invalidateCheckoutGitQueriesForServer } from "@/git/query-keys";
import { queryClient } from "@/query/query-client"; import { queryClient } from "@/query/query-client";
import { mountBrowserAutomationDaemonClientHandler } from "@/browser-automation/handler";
export type HostRuntimeConnectionStatus = "idle" | "connecting" | "online" | "offline" | "error"; export type HostRuntimeConnectionStatus = "idle" | "connecting" | "online" | "offline" | "error";
export type HostRegistryStatus = "loading" | "ready"; export type HostRegistryStatus = "loading" | "ready";
@@ -138,6 +141,11 @@ export interface HostRuntimeControllerDeps {
}>; }>;
getClientId: () => Promise<string>; getClientId: () => Promise<string>;
readInitialConnectionHint?: () => InitialDaemonConnectionHint | null; readInitialConnectionHint?: () => InitialDaemonConnectionHint | null;
mountClientHandlers?: (input: {
client: DaemonClient;
host: HostProfile;
connection: HostConnection;
}) => () => void;
} }
export interface HostRuntimeStorage { export interface HostRuntimeStorage {
@@ -514,6 +522,12 @@ function probeIntervalForConnection(
} }
function createDefaultDeps(): HostRuntimeControllerDeps { function createDefaultDeps(): HostRuntimeControllerDeps {
const desktopBrowserAutomationAvailable =
typeof getDesktopHost()?.browser?.executeAutomationCommand === "function";
const browserAutomationCapabilities = desktopBrowserAutomationAvailable
? { [CLIENT_CAPS.desktopBrowserAutomation]: true }
: undefined;
return { return {
createClient: ({ host, connection, clientId, runtimeGeneration }) => { createClient: ({ host, connection, clientId, runtimeGeneration }) => {
const localTransportFactory = createDesktopLocalDaemonTransportFactory(); const localTransportFactory = createDesktopLocalDaemonTransportFactory();
@@ -523,6 +537,7 @@ function createDefaultDeps(): HostRuntimeControllerDeps {
clientType: "mobile" as const, clientType: "mobile" as const,
appVersion: resolveAppVersion() ?? undefined, appVersion: resolveAppVersion() ?? undefined,
runtimeGeneration, runtimeGeneration,
...(browserAutomationCapabilities ? { capabilities: browserAutomationCapabilities } : {}),
}; };
if (connection.type === "directSocket" || connection.type === "directPipe") { if (connection.type === "directSocket" || connection.type === "directPipe") {
return new DaemonClient({ return new DaemonClient({
@@ -560,8 +575,15 @@ function createDefaultDeps(): HostRuntimeControllerDeps {
connectToDaemon(connection, { connectToDaemon(connection, {
...(host.serverId ? { serverId: host.serverId } : {}), ...(host.serverId ? { serverId: host.serverId } : {}),
...(timeoutMs !== undefined ? { timeoutMs } : {}), ...(timeoutMs !== undefined ? { timeoutMs } : {}),
...(browserAutomationCapabilities ? { capabilities: browserAutomationCapabilities } : {}),
}), }),
getClientId: () => getOrCreateClientId(), getClientId: () => getOrCreateClientId(),
mountClientHandlers: ({ client, host }) => {
if (!browserAutomationCapabilities) {
return () => {};
}
return mountBrowserAutomationDaemonClientHandler(client, { serverId: host.serverId });
},
}; };
} }
@@ -574,6 +596,7 @@ export class HostRuntimeController {
private listeners = new Set<() => void>(); private listeners = new Set<() => void>();
private activeClient: DaemonClient | null = null; private activeClient: DaemonClient | null = null;
private unsubscribeClientStatus: (() => void) | null = null; private unsubscribeClientStatus: (() => void) | null = null;
private unsubscribeClientHandlers: (() => void) | null = null;
private probeIntervalHandle: ReturnType<typeof setInterval> | null = null; private probeIntervalHandle: ReturnType<typeof setInterval> | null = null;
private started = false; private started = false;
private connectionFirstSeenAt = new Map<string, number>(); private connectionFirstSeenAt = new Map<string, number>();
@@ -656,6 +679,10 @@ export class HostRuntimeController {
this.unsubscribeClientStatus(); this.unsubscribeClientStatus();
this.unsubscribeClientStatus = null; this.unsubscribeClientStatus = null;
} }
if (this.unsubscribeClientHandlers) {
this.unsubscribeClientHandlers();
this.unsubscribeClientHandlers = null;
}
if (this.activeClient) { if (this.activeClient) {
const prev = this.activeClient; const prev = this.activeClient;
this.activeClient = null; this.activeClient = null;
@@ -1133,6 +1160,10 @@ export class HostRuntimeController {
this.unsubscribeClientStatus(); this.unsubscribeClientStatus();
this.unsubscribeClientStatus = null; this.unsubscribeClientStatus = null;
} }
if (this.unsubscribeClientHandlers) {
this.unsubscribeClientHandlers();
this.unsubscribeClientHandlers = null;
}
if (this.activeClient) { if (this.activeClient) {
const previousClient = this.activeClient; const previousClient = this.activeClient;
this.activeClient = null; this.activeClient = null;
@@ -1206,6 +1237,8 @@ export class HostRuntimeController {
} }
this.activeClient = client; this.activeClient = client;
this.unsubscribeClientHandlers =
this.deps.mountClientHandlers?.({ client, host: this.host, connection }) ?? null;
this.applyConnectionEvent({ this.applyConnectionEvent({
type: "select_connection", type: "select_connection",
connectionId: connection.id, connectionId: connection.id,
@@ -0,0 +1,70 @@
import React, { useCallback } from "react";
import { Text, View } from "react-native";
import { useMutation } from "@tanstack/react-query";
import { useTranslation } from "react-i18next";
import { Switch } from "@/components/ui/switch";
import { useDaemonConfig } from "@/hooks/use-daemon-config";
import { useHostRuntimeIsConnected } from "@/runtime/host-runtime";
import { settingsStyles } from "@/styles/settings";
import {
createBrowserToolsPatch,
getBrowserToolsCardState,
getBrowserToolsMutationViewState,
} from "./browser-tools-config";
export function BrowserToolsOptInCard({ serverId }: { serverId: string }) {
const { t } = useTranslation();
const isConnected = useHostRuntimeIsConnected(serverId);
const { config, patchConfig } = useDaemonConfig(serverId);
const state = getBrowserToolsCardState({ isConnected, config });
const mutation = useMutation({
mutationFn: async (next: boolean) => {
const result = await patchConfig(createBrowserToolsPatch(next));
if (!result) {
throw new Error(t("workspace.terminal.hostDisconnected"));
}
return result;
},
});
const mutationView = getBrowserToolsMutationViewState({
isPending: mutation.isPending,
error: mutation.error,
});
const handleValueChange = useCallback(
(next: boolean) => {
mutation.mutate(next);
},
[mutation],
);
if (!state.isVisible) return null;
return (
<View style={settingsStyles.card} testID="host-page-browser-tools-card">
<View style={settingsStyles.row}>
<View style={settingsStyles.rowContent}>
<Text style={settingsStyles.rowTitle}>{state.title}</Text>
<Text style={settingsStyles.rowHint}>{state.warning}</Text>
{mutationView.loadingText ? (
<Text style={settingsStyles.rowHint} testID="host-page-browser-tools-loading">
{mutationView.loadingText}
</Text>
) : null}
{mutationView.errorText ? (
<Text style={settingsStyles.rowError} testID="host-page-browser-tools-error">
{mutationView.errorText}
</Text>
) : null}
</View>
<Switch
value={state.isEnabled}
onValueChange={handleValueChange}
disabled={mutationView.isSwitchDisabled}
accessibilityLabel="Enable browser tools"
testID="host-page-browser-tools-switch"
/>
</View>
</View>
);
}
@@ -0,0 +1,70 @@
import type { MutableDaemonConfig } from "@getpaseo/protocol/messages";
import { describe, expect, it } from "vitest";
import {
BROWSER_TOOLS_WARNING,
createBrowserToolsPatch,
getBrowserToolsCardState,
getBrowserToolsMutationViewState,
} from "./browser-tools-config";
function makeConfig(browserToolsEnabled = false): MutableDaemonConfig {
return {
mcp: { injectIntoAgents: false },
browserTools: { enabled: browserToolsEnabled },
providers: {},
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
enableTerminalAgentHooks: false,
appendSystemPrompt: "",
};
}
describe("browser tools opt-in config", () => {
it("shows the card with the logged-in browser state warning when connected", () => {
expect(getBrowserToolsCardState({ isConnected: true, config: makeConfig(false) })).toEqual({
isVisible: true,
isEnabled: false,
title: "Browser tools",
warning: BROWSER_TOOLS_WARNING,
});
});
it("reads enabled state from daemon config", () => {
expect(getBrowserToolsCardState({ isConnected: true, config: makeConfig(true) })).toMatchObject(
{
isEnabled: true,
},
);
});
it("hides the card when the host is disconnected", () => {
expect(
getBrowserToolsCardState({ isConnected: false, config: makeConfig(true) }),
).toMatchObject({
isVisible: false,
});
});
it("writes daemon.browserTools.enabled when toggled", () => {
expect(createBrowserToolsPatch(true)).toEqual({ browserTools: { enabled: true } });
expect(createBrowserToolsPatch(false)).toEqual({ browserTools: { enabled: false } });
});
it("shows loading and disables the toggle while browser tool settings save", () => {
expect(getBrowserToolsMutationViewState({ isPending: true, error: null })).toEqual({
isSwitchDisabled: true,
loadingText: "Updating browser tools…",
errorText: null,
});
});
it("shows the save error when browser tool settings fail", () => {
expect(
getBrowserToolsMutationViewState({ isPending: false, error: new Error("Disk full") }),
).toEqual({
isSwitchDisabled: false,
loadingText: null,
errorText: "Disk full",
});
});
});
@@ -0,0 +1,49 @@
import type { MutableDaemonConfig } from "@getpaseo/protocol/messages";
export const BROWSER_TOOLS_TITLE = "Browser tools";
export const BROWSER_TOOLS_WARNING =
"Allow agents to access and control Paseo desktop browser tabs, including logged-in browser state. Only enable this for agents you trust.";
export interface BrowserToolsCardState {
isVisible: boolean;
isEnabled: boolean;
title: string;
warning: string;
}
export interface BrowserToolsMutationViewState {
isSwitchDisabled: boolean;
loadingText: string | null;
errorText: string | null;
}
export function getBrowserToolsCardState(input: {
isConnected: boolean;
config: MutableDaemonConfig | null;
}): BrowserToolsCardState {
return {
isVisible: input.isConnected,
isEnabled: input.config?.browserTools.enabled === true,
title: BROWSER_TOOLS_TITLE,
warning: BROWSER_TOOLS_WARNING,
};
}
export function createBrowserToolsPatch(enabled: boolean): Partial<MutableDaemonConfig> {
return { browserTools: { enabled } };
}
export function getBrowserToolsMutationViewState(input: {
isPending: boolean;
error: unknown;
}): BrowserToolsMutationViewState {
return {
isSwitchDisabled: input.isPending,
loadingText: input.isPending ? "Updating browser tools…" : null,
errorText: input.error ? toErrorMessage(input.error) : null,
};
}
function toErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
@@ -61,6 +61,7 @@ import { formatLatency } from "@/utils/latency";
import { ICON_SIZE } from "@/styles/theme"; import { ICON_SIZE } from "@/styles/theme";
import type { Theme } from "@/styles/theme"; import type { Theme } from "@/styles/theme";
import { getProviderIcon } from "@/components/provider-icons"; import { getProviderIcon } from "@/components/provider-icons";
import { BrowserToolsOptInCard } from "./browser-tools-card";
const ThemedArrowUp = withUnistyles(ArrowUp); const ThemedArrowUp = withUnistyles(ArrowUp);
const ThemedArrowDown = withUnistyles(ArrowDown); const ThemedArrowDown = withUnistyles(ArrowDown);
@@ -262,6 +263,7 @@ export function HostAgentsPage({ serverId }: { serverId: string }) {
{isConnected ? ( {isConnected ? (
<SettingsSection title={t("settings.hostSections.agents")}> <SettingsSection title={t("settings.hostSections.agents")}>
<InjectPaseoToolsCard serverId={serverId} /> <InjectPaseoToolsCard serverId={serverId} />
<BrowserToolsOptInCard serverId={serverId} />
<AppendSystemPromptCard serverId={serverId} /> <AppendSystemPromptCard serverId={serverId} />
</SettingsSection> </SettingsSection>
) : ( ) : (
@@ -220,6 +220,7 @@ const disabledCodexEntry: ProviderSnapshotEntry = {
function makeConfig(providers: MutableDaemonConfig["providers"] = {}): MutableDaemonConfig { function makeConfig(providers: MutableDaemonConfig["providers"] = {}): MutableDaemonConfig {
return { return {
mcp: { injectIntoAgents: false }, mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers, providers,
metadataGeneration: { providers: [] }, metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
@@ -105,6 +105,7 @@ import type { CheckoutStatusPayload } from "@/git/use-status-query";
import { confirmDialog } from "@/utils/confirm-dialog"; import { confirmDialog } from "@/utils/confirm-dialog";
import { useArchiveAgent } from "@/hooks/use-archive-agent"; import { useArchiveAgent } from "@/hooks/use-archive-agent";
import { useStableEvent } from "@/hooks/use-stable-event"; import { useStableEvent } from "@/hooks/use-stable-event";
import { removeResidentBrowserWebview } from "@/components/browser-webview-resident";
import { createWorkspaceBrowser, useBrowserStore } from "@/stores/browser-store"; import { createWorkspaceBrowser, useBrowserStore } from "@/stores/browser-store";
import { getDesktopHost } from "@/desktop/host"; import { getDesktopHost } from "@/desktop/host";
import { buildProviderCommand } from "@/utils/provider-command-templates"; import { buildProviderCommand } from "@/utils/provider-command-templates";
@@ -304,19 +305,22 @@ function decodeSegment(value: string): string {
function useSyncWorkspaceActiveBrowser(input: { function useSyncWorkspaceActiveBrowser(input: {
workspaceLayout: WorkspaceLayout | null; workspaceLayout: WorkspaceLayout | null;
isRouteFocused: boolean; isRouteFocused: boolean;
workspaceId: string;
}) { }) {
const focusedBrowserId = useMemo( const focusedBrowserId = useMemo(
() => getFocusedBrowserId(input.workspaceLayout), () => getFocusedBrowserId(input.workspaceLayout),
[input.workspaceLayout], [input.workspaceLayout],
); );
const desktopActiveBrowserId = input.isRouteFocused ? focusedBrowserId : null;
useEffect(() => { useEffect(() => {
if (!getIsElectron()) { if (!getIsElectron()) {
return; return;
} }
void getDesktopHost()?.browser?.setWorkspaceActiveBrowser?.(desktopActiveBrowserId); void getDesktopHost()?.browser?.setWorkspaceActiveBrowser?.({
}, [desktopActiveBrowserId]); workspaceId: input.workspaceId,
browserId: focusedBrowserId,
});
}, [focusedBrowserId, input.workspaceId]);
} }
function getFallbackTabOptionLabel( function getFallbackTabOptionLabel(
@@ -1876,7 +1880,11 @@ function WorkspaceScreenContent({
() => (workspaceLayout ? collectAllTabs(workspaceLayout.root) : EMPTY_UI_TABS), () => (workspaceLayout ? collectAllTabs(workspaceLayout.root) : EMPTY_UI_TABS),
[workspaceLayout], [workspaceLayout],
); );
useSyncWorkspaceActiveBrowser({ workspaceLayout, isRouteFocused }); useSyncWorkspaceActiveBrowser({
workspaceLayout,
isRouteFocused,
workspaceId: normalizedWorkspaceId,
});
const openWorkspaceTabInBackground = useWorkspaceLayoutStore( const openWorkspaceTabInBackground = useWorkspaceLayoutStore(
(state) => state.openTabInBackground, (state) => state.openTabInBackground,
); );
@@ -1921,6 +1929,7 @@ function WorkspaceScreenContent({
if (input.target?.kind === "browser") { if (input.target?.kind === "browser") {
const { browserId } = input.target; const { browserId } = input.target;
useBrowserStore.getState().removeBrowser(browserId); useBrowserStore.getState().removeBrowser(browserId);
removeResidentBrowserWebview(browserId);
void getDesktopHost()?.browser?.clearPartition?.(browserId); void getDesktopHost()?.browser?.clearPartition?.(browserId);
} }
closeWorkspaceTab(persistenceKey, normalizedTabId); closeWorkspaceTab(persistenceKey, normalizedTabId);
+5
View File
@@ -63,4 +63,9 @@ export const settingsStyles = StyleSheet.create((theme) => ({
fontSize: theme.fontSize.xs, fontSize: theme.fontSize.xs,
marginTop: theme.spacing[1], marginTop: theme.spacing[1],
}, },
rowError: {
color: theme.colors.statusDanger,
fontSize: theme.fontSize.xs,
marginTop: theme.spacing[1],
},
})); }));
@@ -25,7 +25,7 @@ describe("assistant image metadata", () => {
setAssistantImageMetadata( setAssistantImageMetadata(
{ {
source: "/tmp/paseo-codex-screenshot.png", source: "/tmp/paseo-codex-screenshot.png",
workspaceRoot: "/Users/moboudra/dev/paseo", workspaceRoot: "/workspaces/paseo",
serverId: "server-1", serverId: "server-1",
}, },
{ width: 1200, height: 800 }, { width: 1200, height: 800 },
@@ -5,28 +5,28 @@ describe("buildAbsoluteExplorerPath", () => {
it("builds a POSIX absolute path from a relative explorer path", () => { it("builds a POSIX absolute path from a relative explorer path", () => {
expect( expect(
buildAbsoluteExplorerPath({ buildAbsoluteExplorerPath({
workspaceRoot: "/Users/moboudra/dev/paseo", workspaceRoot: "/workspaces/paseo",
entryPath: "packages/app/src/components/file-explorer-pane.tsx", entryPath: "packages/app/src/components/file-explorer-pane.tsx",
}), }),
).toBe("/Users/moboudra/dev/paseo/packages/app/src/components/file-explorer-pane.tsx"); ).toBe("/workspaces/paseo/packages/app/src/components/file-explorer-pane.tsx");
}); });
it("returns workspace root when entry path points to explorer root", () => { it("returns workspace root when entry path points to explorer root", () => {
expect( expect(
buildAbsoluteExplorerPath({ buildAbsoluteExplorerPath({
workspaceRoot: "/Users/moboudra/dev/paseo", workspaceRoot: "/workspaces/paseo",
entryPath: ".", entryPath: ".",
}), }),
).toBe("/Users/moboudra/dev/paseo"); ).toBe("/workspaces/paseo");
}); });
it("trims trailing separators from workspace root before joining", () => { it("trims trailing separators from workspace root before joining", () => {
expect( expect(
buildAbsoluteExplorerPath({ buildAbsoluteExplorerPath({
workspaceRoot: "/Users/moboudra/dev/paseo/", workspaceRoot: "/workspaces/paseo/",
entryPath: "README.md", entryPath: "README.md",
}), }),
).toBe("/Users/moboudra/dev/paseo/README.md"); ).toBe("/workspaces/paseo/README.md");
}); });
it("builds a Windows absolute path with backslash separators", () => { it("builds a Windows absolute path with backslash separators", () => {
@@ -41,7 +41,7 @@ describe("buildAbsoluteExplorerPath", () => {
it("passes through an already-absolute entry path", () => { it("passes through an already-absolute entry path", () => {
expect( expect(
buildAbsoluteExplorerPath({ buildAbsoluteExplorerPath({
workspaceRoot: "/Users/moboudra/dev/paseo", workspaceRoot: "/workspaces/paseo",
entryPath: "/tmp/another/location.txt", entryPath: "/tmp/another/location.txt",
}), }),
).toBe("/tmp/another/location.txt"); ).toBe("/tmp/another/location.txt");
+2 -2
View File
@@ -45,8 +45,8 @@ describe("workspace route parsing", () => {
}); });
it("decodes non-canonical base64url workspace IDs used by older links", () => { it("decodes non-canonical base64url workspace IDs used by older links", () => {
expect(decodeWorkspaceIdFromPathSegment("L1VzZXJzL21vYm91ZHJhL2Rldi9wYXNlby")).toBe( expect(decodeWorkspaceIdFromPathSegment("L2hvbWUvdXNlci9kZXYvcGFzZW8")).toBe(
"/Users/moboudra/dev/paseo", "/home/user/dev/paseo",
); );
}); });
@@ -97,6 +97,7 @@ export class DaemonConnectionTestError extends Error {
export async function buildClientConfig( export async function buildClientConfig(
connection: HostConnection, connection: HostConnection,
serverId?: string, serverId?: string,
options?: { capabilities?: DaemonClientConfig["capabilities"] },
deps: Pick< deps: Pick<
DaemonConnectionDependencies<DaemonProbeClient>, DaemonConnectionDependencies<DaemonProbeClient>,
"getClientId" | "resolveAppVersion" | "createLocalTransportFactory" | "buildLocalTransportUrl" "getClientId" | "resolveAppVersion" | "createLocalTransportFactory" | "buildLocalTransportUrl"
@@ -110,6 +111,7 @@ export async function buildClientConfig(
appVersion: deps.resolveAppVersion() ?? undefined, appVersion: deps.resolveAppVersion() ?? undefined,
suppressSendErrors: true, suppressSendErrors: true,
reconnect: { enabled: false }, reconnect: { enabled: false },
...(options?.capabilities ? { capabilities: options.capabilities } : {}),
...((connection.type === "directSocket" || connection.type === "directPipe") && ...((connection.type === "directSocket" || connection.type === "directPipe") &&
localTransportFactory localTransportFactory
? { transportFactory: localTransportFactory } ? { transportFactory: localTransportFactory }
@@ -221,6 +223,7 @@ export function connectAndProbe(
interface ProbeOptions { interface ProbeOptions {
serverId?: string; serverId?: string;
timeoutMs?: number; timeoutMs?: number;
capabilities?: DaemonClientConfig["capabilities"];
} }
function resolveTimeout(connection: HostConnection, options?: ProbeOptions): number { function resolveTimeout(connection: HostConnection, options?: ProbeOptions): number {
@@ -242,6 +245,6 @@ export async function connectToDaemon(
options?: ProbeOptions, options?: ProbeOptions,
deps: DaemonConnectionDependencies<DaemonProbeClient> = defaultDaemonConnectionDependencies, deps: DaemonConnectionDependencies<DaemonProbeClient> = defaultDaemonConnectionDependencies,
): Promise<{ client: DaemonProbeClient; serverId: string; hostname: string | null }> { ): Promise<{ client: DaemonProbeClient; serverId: string; hostname: string | null }> {
const config = await buildClientConfig(connection, options?.serverId, deps); const config = await buildClientConfig(connection, options?.serverId, options, deps);
return connectAndProbe(config, resolveTimeout(connection, options), deps); return connectAndProbe(config, resolveTimeout(connection, options), deps);
} }
+31
View File
@@ -1,5 +1,6 @@
// @ts-nocheck // @ts-nocheck
import { vi } from "vitest"; import { vi } from "vitest";
import React from "react";
const globalWithTestShims = globalThis as typeof globalThis & Record<string, unknown>; const globalWithTestShims = globalThis as typeof globalThis & Record<string, unknown>;
@@ -94,3 +95,33 @@ vi.mock("react-native-svg", () => {
vi.mock("expo-linking", () => ({ vi.mock("expo-linking", () => ({
openURL: vi.fn().mockResolvedValue(undefined), openURL: vi.fn().mockResolvedValue(undefined),
})); }));
const RouterPassthrough = ({ children }: { children?: React.ReactNode }) => children;
vi.mock("expo-router", () => ({
Redirect: () => null,
Stack: Object.assign(RouterPassthrough, {
Screen: () => null,
Protected: RouterPassthrough,
}),
router: {
back: vi.fn(),
canGoBack: vi.fn(() => false),
navigate: vi.fn(),
push: vi.fn(),
replace: vi.fn(),
setParams: vi.fn(),
},
useGlobalSearchParams: vi.fn(() => ({})),
useLocalSearchParams: vi.fn(() => ({})),
usePathname: vi.fn(() => "/"),
useRootNavigationState: vi.fn(() => ({ key: "root" })),
useRouter: vi.fn(() => ({
back: vi.fn(),
canGoBack: vi.fn(() => false),
navigate: vi.fn(),
push: vi.fn(),
replace: vi.fn(),
setParams: vi.fn(),
})),
}));
+91
View File
@@ -1,6 +1,7 @@
import { afterEach, expect, expectTypeOf, test, vi } from "vitest"; import { afterEach, expect, expectTypeOf, test, vi } from "vitest";
import { z } from "zod"; import { z } from "zod";
import { DaemonClient, type DaemonTransport, type Logger } from "./daemon-client"; import { DaemonClient, type DaemonTransport, type Logger } from "./daemon-client";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
import { import {
decodeFileTransferFrame, decodeFileTransferFrame,
encodeFileTransferFrame, encodeFileTransferFrame,
@@ -136,6 +137,58 @@ afterEach(async () => {
await Promise.all(clients.map((client) => client.close())); await Promise.all(clients.map((client) => client.close()));
clients.length = 0; clients.length = 0;
vi.useRealTimers(); vi.useRealTimers();
vi.unstubAllGlobals();
});
test("does not infer browser automation capabilities from Electron runtime", async () => {
vi.stubGlobal("navigator", {
userAgent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Paseo/0.1.89 Chrome/146 Electron/41.2.0 Safari/537.36",
});
const mock = createMockTransport();
const client = new DaemonClient({
url: "ws://test",
clientId: "electron_unit_test",
transportFactory: () => mock.transport,
reconnect: { enabled: false },
});
clients.push(client);
const connectPromise = client.connect();
mock.triggerOpen({ preserveSent: true });
await connectPromise;
const hello = z
.object({
type: z.literal("hello"),
capabilities: z.record(z.unknown()),
})
.parse(JSON.parse(assertStr(mock.sent[0])));
expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBeUndefined();
});
test("advertises consumer-provided browser automation capabilities", async () => {
const mock = createMockTransport();
const client = new DaemonClient({
url: "ws://test",
clientId: "browser_capability_unit_test",
transportFactory: () => mock.transport,
reconnect: { enabled: false },
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
});
clients.push(client);
const connectPromise = client.connect();
mock.triggerOpen({ preserveSent: true });
await connectPromise;
const hello = z
.object({
type: z.literal("hello"),
capabilities: z.record(z.unknown()),
})
.parse(JSON.parse(assertStr(mock.sent[0])));
expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBe(true);
}); });
const noopLogger: Logger = { const noopLogger: Logger = {
@@ -453,6 +506,7 @@ test("advertises client capabilities in hello", async () => {
logger, logger,
reconnect: { enabled: false }, reconnect: { enabled: false },
transportFactory: () => mock.transport, transportFactory: () => mock.transport,
capabilities: { desktop_browser_automation: true },
}); });
clients.push(client); clients.push(client);
@@ -470,6 +524,43 @@ test("advertises client capabilities in hello", async () => {
custom_mode_icons: true, custom_mode_icons: true,
reasoning_merge_enum: true, reasoning_merge_enum: true,
terminal_reflowable_snapshot: true, terminal_reflowable_snapshot: true,
desktop_browser_automation: true,
},
});
});
test("sends typed browser automation execute responses", async () => {
const logger = createMockLogger();
const mock = createMockTransport();
const client = new DaemonClient({
url: "ws://test",
clientId: "clsk_unit_test",
logger,
reconnect: { enabled: false },
transportFactory: () => mock.transport,
});
clients.push(client);
const connectPromise = client.connect();
mock.triggerOpen();
await connectPromise;
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: true,
result: { command: "list_tabs", tabs: [] },
},
});
expect(parseSentFrame(mock.sent[0])).toEqual({
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: true,
result: { command: "list_tabs", tabs: [] },
}, },
}); });
}); });
+13 -1
View File
@@ -1,5 +1,5 @@
import type { z } from "zod"; import type { z } from "zod";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; import { CLIENT_CAPS, type ClientCapability } from "@getpaseo/protocol/client-capabilities";
import { import {
AgentCreateFailedStatusPayloadSchema, AgentCreateFailedStatusPayloadSchema,
AgentCreatedStatusPayloadSchema, AgentCreatedStatusPayloadSchema,
@@ -119,6 +119,10 @@ import {
} from "./daemon-client-transport.js"; } from "./daemon-client-transport.js";
import { DaemonClientRuntimeMetrics } from "./daemon-client-runtime-metrics.js"; import { DaemonClientRuntimeMetrics } from "./daemon-client-runtime-metrics.js";
import { TerminalStreamRouter, type TerminalStreamEvent } from "./terminal-stream-router.js"; import { TerminalStreamRouter, type TerminalStreamEvent } from "./terminal-stream-router.js";
import type {
BrowserAutomationExecuteRequest,
BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
export interface Logger { export interface Logger {
debug(obj: object, msg?: string): void; debug(obj: object, msg?: string): void;
@@ -221,6 +225,8 @@ export type DaemonEvent =
| { type: "error"; message: string }; | { type: "error"; message: string };
export type DaemonEventHandler = (event: DaemonEvent) => void; export type DaemonEventHandler = (event: DaemonEvent) => void;
export type BrowserAutomationExecuteRequestMessage = BrowserAutomationExecuteRequest;
export type BrowserAutomationExecuteResponseMessage = BrowserAutomationExecuteResponse;
export interface DaemonClientConfig { export interface DaemonClientConfig {
url: string; url: string;
@@ -246,6 +252,7 @@ export interface DaemonClientConfig {
}; };
runtimeMetricsIntervalMs?: number; runtimeMetricsIntervalMs?: number;
runtimeMetricsWindowMs?: number; runtimeMetricsWindowMs?: number;
capabilities?: Partial<Record<ClientCapability, boolean>>;
} }
export interface SendMessageOptions { export interface SendMessageOptions {
@@ -3850,6 +3857,10 @@ export class DaemonClient {
}); });
} }
sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void {
this.sendSessionMessageStrict(response);
}
async readProjectConfig(repoRoot: string, requestId?: string): Promise<ReadProjectConfigPayload> { async readProjectConfig(repoRoot: string, requestId?: string): Promise<ReadProjectConfigPayload> {
return this.sendCorrelatedSessionRequest({ return this.sendCorrelatedSessionRequest({
requestId, requestId,
@@ -4648,6 +4659,7 @@ export class DaemonClient {
[CLIENT_CAPS.customModeIcons]: true, [CLIENT_CAPS.customModeIcons]: true,
[CLIENT_CAPS.reasoningMergeEnum]: true, [CLIENT_CAPS.reasoningMergeEnum]: true,
[CLIENT_CAPS.terminalReflowableSnapshot]: true, [CLIENT_CAPS.terminalReflowableSnapshot]: true,
...this.config.capabilities,
}, },
...(this.config.appVersion ? { appVersion: this.config.appVersion } : {}), ...(this.config.appVersion ? { appVersion: this.config.appVersion } : {}),
}), }),
+2
View File
@@ -674,6 +674,7 @@ test("config actions delegate to existing daemon config RPCs", async () => {
config: { config: {
mcp: { injectIntoAgents: true }, mcp: { injectIntoAgents: true },
providers: {}, providers: {},
browserTools: { enabled: false },
metadataGeneration: { providers: [] }, metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
enableTerminalAgentHooks: false, enableTerminalAgentHooks: false,
@@ -728,6 +729,7 @@ test("config actions delegate to existing daemon config RPCs", async () => {
enabled: false, enabled: false,
}, },
}, },
browserTools: { enabled: false },
metadataGeneration: { providers: [] }, metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
enableTerminalAgentHooks: false, enableTerminalAgentHooks: false,
+2
View File
@@ -30,6 +30,8 @@ export { DaemonClient };
export type { export type {
DaemonClientConfig, DaemonClientConfig,
DaemonEvent, DaemonEvent,
BrowserAutomationExecuteRequestMessage,
BrowserAutomationExecuteResponseMessage,
WebSocketFactory, WebSocketFactory,
WebSocketLike, WebSocketLike,
} from "./daemon-client.js"; } from "./daemon-client.js";
-2
View File
@@ -19,7 +19,6 @@ if (!Number.isInteger(expoPort) || expoPort <= 0) {
} }
const expoDevUrl = process.env.EXPO_DEV_URL || `http://localhost:${expoPort}`; const expoDevUrl = process.env.EXPO_DEV_URL || `http://localhost:${expoPort}`;
const daemonEndpoint = process.env.PASEO_DAEMON_ENDPOINT || process.env.PASEO_LISTEN || "";
const colorEnv = { const colorEnv = {
FORCE_COLOR: process.env.FORCE_COLOR || "1", FORCE_COLOR: process.env.FORCE_COLOR || "1",
npm_config_color: process.env.npm_config_color || "always", npm_config_color: process.env.npm_config_color || "always",
@@ -164,7 +163,6 @@ spawnChild("metro", "npx", ["expo", "start", "--port", String(expoPort)], {
BROWSER: "none", BROWSER: "none",
APP_VARIANT: "development", APP_VARIANT: "development",
PASEO_WEB_PLATFORM: "electron", PASEO_WEB_PLATFORM: "electron",
EXPO_PUBLIC_LOCAL_DAEMON: daemonEndpoint,
}, },
}); });
@@ -65,6 +65,16 @@ describe("passthrough CLI", () => {
).toBeNull(); ).toBeNull();
}); });
it("ignores Electron remote debugging switches", () => {
expect(
parsePassthroughCliArgs({
argv: ["/usr/bin/Paseo", "--remote-debugging-port=9233"],
isDefaultApp: false,
forceCli: false,
}),
).toBeNull();
});
it("preserves CLI flags for direct app invocations", () => { it("preserves CLI flags for direct app invocations", () => {
expect( expect(
parsePassthroughCliArgs({ parsePassthroughCliArgs({
@@ -2,7 +2,7 @@ import { pathToFileURL } from "node:url";
import { resolvePassthroughCliEntrypoint } from "./entrypoints.js"; import { resolvePassthroughCliEntrypoint } from "./entrypoints.js";
const DESKTOP_CLI_ENV = "PASEO_DESKTOP_CLI"; const DESKTOP_CLI_ENV = "PASEO_DESKTOP_CLI";
const IGNORED_ARG_PREFIXES = ["-psn_", "--no-sandbox"]; const IGNORED_ARG_PREFIXES = ["-psn_", "--no-sandbox", "--remote-debugging-port="];
export type PassthroughCliRunner = (argv: string[]) => Promise<number>; export type PassthroughCliRunner = (argv: string[]) => Promise<number>;
@@ -0,0 +1,18 @@
import { describe, expect, it } from "vitest";
import { sanitizeDownloadFileName } from "./ipc.js";
describe("browser automation IPC", () => {
it("strips directories from agent-supplied download filenames", () => {
expect(
sanitizeDownloadFileName({
url: "https://example.com/fallback.txt",
fileName: "../../.ssh/authorized_keys",
}),
).toBe("authorized_keys");
});
it("falls back to a safe filename when the URL has no basename", () => {
expect(sanitizeDownloadFileName({ url: "https://example.com/" })).toBe("download");
});
});
@@ -0,0 +1,185 @@
import { mkdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import type { WebContents } from "electron";
import { ipcMain } from "electron";
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type {
BrowserAutomationConsoleLogEntry,
BrowserAutomationCookieEntry,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { TabContents, BrowserRegistry } from "./service.js";
import { executeAutomationCommand } from "./service.js";
import {
listRegisteredPaseoBrowserIds,
listRegisteredPaseoBrowserIdsForWorkspace,
getPaseoBrowserWebContents,
getWorkspaceActivePaseoBrowserWebContents,
getWorkspaceActivePaseoBrowserId,
getAgentActivePaseoBrowserId,
getPaseoBrowserWorkspaceId,
} from "../browser-webviews/index.js";
const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
const observedContentsIds = new Set<number>();
interface IpcHandlerRegistry {
handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void;
}
function adaptWebContents(contents: WebContents): TabContents {
observeConsoleMessages(contents);
return {
id: contents.id,
getURL: () => contents.getURL(),
getTitle: () => contents.getTitle(),
canGoBack: () => contents.canGoBack(),
canGoForward: () => contents.canGoForward(),
isLoading: () => contents.isLoading(),
isDestroyed: () => contents.isDestroyed(),
executeJavaScript: (code: string) => contents.executeJavaScript(code),
loadURL: (url: string) => contents.loadURL(url),
goBack: () => contents.goBack(),
goForward: () => contents.goForward(),
reload: () => contents.reload(),
capturePage: () => contents.capturePage(),
getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
getCookies: async (url: string) =>
(await contents.session.cookies.get({ url })).map(normalizeCookie),
sendDebugCommand: async (command: string, params?: Record<string, unknown>) => {
if (!contents.debugger.isAttached()) {
contents.debugger.attach("1.3");
}
return contents.debugger.sendCommand(command, params ?? {});
},
printToPDF: async (options?: Record<string, unknown>) => contents.printToPDF(options ?? {}),
downloadURL: (input) => downloadWithContents(contents, input),
};
}
function downloadWithContents(
contents: WebContents,
input: { url: string; fileName?: string },
): Promise<{ filePath: string; totalBytes?: number; state: string }> {
const downloadDir = join(tmpdir(), "paseo-browser-downloads");
mkdirSync(downloadDir, { recursive: true });
const filePath = join(downloadDir, sanitizeDownloadFileName(input));
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
contents.session.off("will-download", onDownload);
reject(new Error(`Timed out waiting for browser download: ${input.url}`));
}, 30_000);
function onDownload(_event: Electron.Event, item: Electron.DownloadItem): void {
if (item.getURL() !== input.url) {
return;
}
clearTimeout(timeout);
contents.session.off("will-download", onDownload);
item.setSavePath(filePath);
item.once("done", (_doneEvent, state) => {
resolve({ filePath, totalBytes: item.getTotalBytes(), state });
});
}
contents.session.on("will-download", onDownload);
contents.downloadURL(input.url);
});
}
export function sanitizeDownloadFileName(input: { url: string; fileName?: string }): string {
const requestedName = input.fileName ?? basename(new URL(input.url).pathname);
return basename(requestedName) || "download";
}
function normalizeCookie(cookie: Electron.Cookie): BrowserAutomationCookieEntry {
return {
name: cookie.name,
value: cookie.value,
...(cookie.domain ? { domain: cookie.domain } : {}),
...(cookie.path ? { path: cookie.path } : {}),
secure: cookie.secure,
httpOnly: cookie.httpOnly,
...(typeof cookie.expirationDate === "number" ? { expirationDate: cookie.expirationDate } : {}),
};
}
function observeConsoleMessages(contents: WebContents): void {
if (observedContentsIds.has(contents.id)) {
return;
}
observedContentsIds.add(contents.id);
contents.on("console-message", (_event, level, message, line, sourceId) => {
const entry = normalizeConsoleMessage({ level, message, line, sourceId });
const messages = consoleMessagesByContentsId.get(contents.id) ?? [];
messages.push(entry);
consoleMessagesByContentsId.set(contents.id, messages.slice(-MAX_CONSOLE_MESSAGES_PER_TAB));
});
contents.once("destroyed", () => {
observedContentsIds.delete(contents.id);
consoleMessagesByContentsId.delete(contents.id);
});
}
function normalizeConsoleMessage(input: {
level: unknown;
message: unknown;
line: unknown;
sourceId: unknown;
}): BrowserAutomationConsoleLogEntry {
return {
level: typeof input.level === "string" ? input.level : String(input.level ?? "log"),
message: typeof input.message === "string" ? input.message : String(input.message ?? ""),
...(typeof input.sourceId === "string" && input.sourceId.length > 0
? { source: input.sourceId }
: {}),
...(typeof input.line === "number" ? { line: input.line } : {}),
timestamp: Date.now(),
};
}
function createRegistry(): BrowserRegistry {
return {
listRegisteredBrowserIds: listRegisteredPaseoBrowserIds,
listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace,
getTabContents(browserId: string): TabContents | null {
const contents = getPaseoBrowserWebContents(browserId);
return contents ? adaptWebContents(contents) : null;
},
getBrowserWorkspaceId: getPaseoBrowserWorkspaceId,
getWorkspaceActiveTabContents(workspaceId: string): TabContents | null {
const contents = getWorkspaceActivePaseoBrowserWebContents(workspaceId);
return contents ? adaptWebContents(contents) : null;
},
getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId,
getAgentActiveBrowserId: getAgentActivePaseoBrowserId,
};
}
export function registerBrowserAutomationIpc(options?: { ipc?: IpcHandlerRegistry }): void {
const ipc = options?.ipc ?? ipcMain;
const registry = createRegistry();
ipc.handle("paseo:browser:execute-automation-command", async (_event, rawRequest: unknown) => {
const parsed = BrowserAutomationExecuteRequestSchema.safeParse(rawRequest);
if (!parsed.success) {
return {
requestId: readRequestId(rawRequest),
ok: false as const,
error: {
code: "browser_unsupported" as const,
message: `Invalid automation request: ${parsed.error.message}`,
retryable: false,
},
};
}
return executeAutomationCommand(parsed.data, registry);
});
}
function readRequestId(rawRequest: unknown): string {
if (typeof rawRequest !== "object" || rawRequest === null || Array.isArray(rawRequest)) {
return "unknown";
}
const requestId = (rawRequest as Record<string, unknown>).requestId;
return typeof requestId === "string" && requestId.length > 0 ? requestId : "unknown";
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,79 @@
import { describe, expect, it } from "vitest";
import { BrowserSnapshotEngine, type SnapshotPage } from "./snapshot-engine.js";
class SnapshotFixture implements SnapshotPage {
public currentUrl = "https://example.com/form";
public actionResult: unknown = true;
public getURL(): string {
return this.currentUrl;
}
public async executeJavaScript(code: string): Promise<unknown> {
if (code.includes("CANDIDATE_SELECTOR")) {
return JSON.stringify([
{
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name", type: "text" },
},
{
role: "button",
tagName: "button",
text: "Drop",
selector: "#drop",
attributes: { id: "drop" },
},
]);
}
return this.actionResult;
}
}
describe("BrowserSnapshotEngine", () => {
it("treats a false result from a ref action script as a stale ref", async () => {
const page = new SnapshotFixture();
const engine = new BrowserSnapshotEngine();
await engine.snapshot({ browserId: "browser-1", page });
page.actionResult = false;
await expect(engine.click({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({
ok: false,
reason: "stale_ref",
});
await expect(engine.focus({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({
ok: false,
reason: "stale_ref",
});
});
it("treats a false result from optional ref text/key actions as a stale ref", async () => {
const page = new SnapshotFixture();
const engine = new BrowserSnapshotEngine();
await engine.snapshot({ browserId: "browser-1", page });
page.actionResult = false;
await expect(
engine.typeText({ browserId: "browser-1", page, ref: "@e1", text: "Ada" }),
).resolves.toEqual({ ok: false, reason: "stale_ref" });
await expect(
engine.keypress({ browserId: "browser-1", page, ref: "@e1", key: "Enter" }),
).resolves.toEqual({ ok: false, reason: "stale_ref" });
});
it("treats a false result from drag as a stale ref", async () => {
const page = new SnapshotFixture();
const engine = new BrowserSnapshotEngine();
await engine.snapshot({ browserId: "browser-1", page });
page.actionResult = false;
await expect(
engine.drag({ browserId: "browser-1", page, sourceRef: "@e1", targetRef: "@e2" }),
).resolves.toEqual({ ok: false, reason: "stale_ref" });
});
});
@@ -0,0 +1,570 @@
export interface SnapshotPage {
getURL(): string;
executeJavaScript(code: string): Promise<unknown>;
}
export interface BrowserSnapshotElement extends RawSnapshotElement {
ref: string;
}
interface RawSnapshotElement {
role: string;
tagName: string;
text: string;
selector: string;
attributes: Record<string, string>;
}
interface BrowserRefState {
nextRefNumber: number;
url: string;
refs: Map<string, RawSnapshotElement>;
}
export type BrowserRefActionResult =
| { ok: true }
| { ok: false; reason: "stale_ref" | "missing_ref" };
type BrowserRefFailure = Extract<BrowserRefActionResult, { ok: false }>;
type BrowserRefResolveResult = { ok: true; element: RawSnapshotElement } | BrowserRefFailure;
export class BrowserSnapshotEngine {
private readonly statesByBrowserId = new Map<string, BrowserRefState>();
async snapshot(input: {
browserId: string;
page: SnapshotPage;
}): Promise<BrowserSnapshotElement[]> {
const rawElements = parseRawSnapshotElements(
await input.page.executeJavaScript(SNAPSHOT_SCRIPT),
);
const state = {
nextRefNumber: 1,
url: input.page.getURL(),
refs: new Map<string, RawSnapshotElement>(),
};
const elements = rawElements.map((element) => {
const ref = `@e${state.nextRefNumber++}`;
state.refs.set(ref, element);
return {
ref,
role: element.role,
tagName: element.tagName,
text: element.text,
selector: element.selector,
attributes: element.attributes,
};
});
this.statesByBrowserId.set(input.browserId, state);
return elements;
}
async click(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildClickScript(selector));
}
async fill(input: {
browserId: string;
page: SnapshotPage;
ref: string;
value: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildFillScript(selector, input.value));
}
async typeText(input: {
browserId: string;
page: SnapshotPage;
ref?: string;
text: string;
}): Promise<BrowserRefActionResult> {
const selector = this.resolveOptionalRef(input);
if (!selector.ok) {
return selector;
}
const result = await input.page.executeJavaScript(
buildTypeScript(selector.selector, input.text),
);
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
async keypress(input: {
browserId: string;
page: SnapshotPage;
ref?: string;
key: string;
}): Promise<BrowserRefActionResult> {
const selector = this.resolveOptionalRef(input);
if (!selector.ok) {
return selector;
}
const result = await input.page.executeJavaScript(
buildKeypressScript(selector.selector, input.key),
);
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
async focus(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildFocusScript(selector));
}
async clear(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildClearScript(selector));
}
async check(input: {
browserId: string;
page: SnapshotPage;
ref: string;
checked: boolean;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildCheckScript(selector, input.checked));
}
async select(input: {
browserId: string;
page: SnapshotPage;
ref: string;
value: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildSelectScript(selector, input.value));
}
async hover(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildHoverScript(selector));
}
async drag(input: {
browserId: string;
page: SnapshotPage;
sourceRef: string;
targetRef: string;
}): Promise<BrowserRefActionResult> {
const source = this.resolveRef({
browserId: input.browserId,
page: input.page,
ref: input.sourceRef,
});
if (!source.ok) {
return source;
}
const target = this.resolveRef({
browserId: input.browserId,
page: input.page,
ref: input.targetRef,
});
if (!target.ok) {
return target;
}
const result = await input.page.executeJavaScript(
buildDragScript(source.element.selector, target.element.selector),
);
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
clearBrowser(browserId: string): void {
this.statesByBrowserId.delete(browserId);
}
selectorForRef(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): { ok: true; selector: string } | BrowserRefFailure {
const resolved = this.resolveRef(input);
if (!resolved.ok) {
return resolved;
}
return { ok: true, selector: resolved.element.selector };
}
private async runRefScript(
input: { browserId: string; page: SnapshotPage; ref: string },
buildScript: (selector: string) => string,
): Promise<BrowserRefActionResult> {
const resolved = this.resolveRef(input);
if (!resolved.ok) {
return resolved;
}
const result = await input.page.executeJavaScript(buildScript(resolved.element.selector));
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
private resolveRef(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): BrowserRefResolveResult {
const state = this.statesByBrowserId.get(input.browserId);
if (!state || state.url !== input.page.getURL()) {
return { ok: false, reason: "stale_ref" };
}
const element = state.refs.get(input.ref);
if (!element) {
return { ok: false, reason: "missing_ref" };
}
return { ok: true, element };
}
private resolveOptionalRef(input: {
browserId: string;
page: SnapshotPage;
ref?: string;
}): { ok: true; selector: string | undefined } | BrowserRefFailure {
if (!input.ref) {
return { ok: true, selector: undefined };
}
const resolved = this.resolveRef({
browserId: input.browserId,
page: input.page,
ref: input.ref,
});
if (!resolved.ok) {
return resolved;
}
return { ok: true, selector: resolved.element.selector };
}
}
function buildClickScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView({ block: 'center', inline: 'center' });
element.click();
return true;
})()`;
}
function buildFillScript(selector: string, value: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView({ block: 'center', inline: 'center' });
element.focus();
const nextValue = ${JSON.stringify(value)};
if ('value' in element) {
element.value = nextValue;
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
element.textContent = nextValue;
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: nextValue }));
return true;
})()`;
}
function buildTypeScript(selector: string | undefined, text: string): string {
return String.raw`(() => {
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
const text = ${JSON.stringify(text)};
if ('value' in element) {
element.value = String(element.value || '') + text;
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
element.textContent = String(element.textContent || '') + text;
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
return true;
})()`;
}
function buildKeypressScript(selector: string | undefined, key: string): string {
return String.raw`(() => {
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
if (!element) return false;
element.focus?.();
const key = ${JSON.stringify(key)};
const eventInit = { bubbles: true, cancelable: true, key };
element.dispatchEvent(new KeyboardEvent('keydown', eventInit));
element.dispatchEvent(new KeyboardEvent('keypress', eventInit));
element.dispatchEvent(new KeyboardEvent('keyup', eventInit));
return true;
})()`;
}
function buildFocusScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
return document.activeElement === element;
})()`;
}
function buildClearScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
if ('value' in element) {
element.value = '';
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
element.textContent = '';
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
return true;
})()`;
}
function buildCheckScript(selector: string, checked: boolean): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
const nextChecked = ${JSON.stringify(checked)};
if ('checked' in element) {
element.checked = nextChecked;
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
if (element.getAttribute('role') === 'checkbox' || element.getAttribute('role') === 'radio') {
element.setAttribute('aria-checked', String(nextChecked));
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
return false;
})()`;
}
function buildSelectScript(selector: string, value: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
const nextValue = ${JSON.stringify(value)};
if ('value' in element) {
element.value = nextValue;
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
return false;
})()`;
}
function buildHoverScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
const rect = element.getBoundingClientRect();
const eventInit = {
bubbles: true,
cancelable: true,
clientX: rect.left + rect.width / 2,
clientY: rect.top + rect.height / 2,
screenX: window.screenX + rect.left + rect.width / 2,
screenY: window.screenY + rect.top + rect.height / 2,
view: window,
};
element.dispatchEvent(new MouseEvent('mouseover', eventInit));
element.dispatchEvent(new MouseEvent('mouseenter', eventInit));
element.dispatchEvent(new MouseEvent('mousemove', eventInit));
return true;
})()`;
}
function buildDragScript(sourceSelector: string, targetSelector: string): string {
return String.raw`(() => {
const source = document.querySelector(${JSON.stringify(sourceSelector)});
const target = document.querySelector(${JSON.stringify(targetSelector)});
if (!source || !target) return false;
source.scrollIntoView?.({ block: 'center', inline: 'center' });
target.scrollIntoView?.({ block: 'center', inline: 'center' });
const data = new DataTransfer();
const sourceRect = source.getBoundingClientRect();
const targetRect = target.getBoundingClientRect();
function eventInit(rect) {
return {
bubbles: true,
cancelable: true,
clientX: rect.left + rect.width / 2,
clientY: rect.top + rect.height / 2,
screenX: window.screenX + rect.left + rect.width / 2,
screenY: window.screenY + rect.top + rect.height / 2,
dataTransfer: data,
view: window,
};
}
source.dispatchEvent(new MouseEvent('mousedown', eventInit(sourceRect)));
source.dispatchEvent(new DragEvent('dragstart', eventInit(sourceRect)));
target.dispatchEvent(new DragEvent('dragenter', eventInit(targetRect)));
target.dispatchEvent(new DragEvent('dragover', eventInit(targetRect)));
target.dispatchEvent(new DragEvent('drop', eventInit(targetRect)));
source.dispatchEvent(new DragEvent('dragend', eventInit(sourceRect)));
target.dispatchEvent(new MouseEvent('mouseup', eventInit(targetRect)));
return true;
})()`;
}
function parseRawSnapshotElements(value: unknown): RawSnapshotElement[] {
const parsed = typeof value === "string" ? JSON.parse(value) : value;
if (!Array.isArray(parsed)) {
return [];
}
return parsed.flatMap((item): RawSnapshotElement[] => {
if (!item || typeof item !== "object") {
return [];
}
const record = item as Record<string, unknown>;
const selector = readString(record.selector);
if (!selector) {
return [];
}
return [
{
role: readString(record.role) || "generic",
tagName: (readString(record.tagName) || "element").toLowerCase(),
text: readString(record.text) || "",
selector,
attributes: readAttributes(record.attributes),
},
];
});
}
function readString(value: unknown): string | null {
return typeof value === "string" ? value : null;
}
function readAttributes(value: unknown): Record<string, string> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return {};
}
const result: Record<string, string> = {};
for (const [key, attributeValue] of Object.entries(value)) {
if (typeof attributeValue === "string") {
result[key] = attributeValue;
}
}
return result;
}
const SNAPSHOT_SCRIPT = String.raw`(() => {
const MAX_ELEMENTS = 200;
const CANDIDATE_SELECTOR = [
'a[href]',
'button',
'input',
'textarea',
'select',
'summary',
'[role]',
'[tabindex]:not([tabindex="-1"])',
'[contenteditable=""]',
'[contenteditable="true"]'
].join(',');
function cssEscape(value) {
if (window.CSS && typeof window.CSS.escape === 'function') {
return window.CSS.escape(value);
}
return String(value).replace(/[^a-zA-Z0-9_-]/g, '\\$&');
}
function isVisible(element) {
const style = window.getComputedStyle(element);
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
return false;
}
const rect = element.getBoundingClientRect();
return rect.width > 0 && rect.height > 0;
}
function roleFor(element) {
const explicit = element.getAttribute('role');
if (explicit) return explicit;
const tag = element.tagName.toLowerCase();
if (tag === 'a') return 'link';
if (tag === 'button') return 'button';
if (tag === 'select') return 'combobox';
if (tag === 'textarea') return 'textbox';
if (tag === 'summary') return 'button';
if (tag === 'input') {
const type = (element.getAttribute('type') || 'text').toLowerCase();
if (type === 'checkbox') return 'checkbox';
if (type === 'radio') return 'radio';
if (type === 'button' || type === 'submit' || type === 'reset') return 'button';
return 'textbox';
}
return 'generic';
}
function textFor(element) {
const tag = element.tagName.toLowerCase();
const pieces = [
element.getAttribute('aria-label'),
element.getAttribute('alt'),
element.getAttribute('title'),
tag === 'input' ? element.getAttribute('placeholder') : null,
tag === 'input' || tag === 'textarea' ? element.value : null,
element.innerText,
element.textContent
];
const text = pieces.find((piece) => typeof piece === 'string' && piece.trim().length > 0);
return (text || '').replace(/\s+/g, ' ').trim().slice(0, 300);
}
function selectorFor(element) {
if (element.id) return '#' + cssEscape(element.id);
const parts = [];
let current = element;
while (current && current.nodeType === Node.ELEMENT_NODE && current !== document.body) {
const tag = current.tagName.toLowerCase();
const parent = current.parentElement;
if (!parent) break;
const siblings = Array.from(parent.children).filter((sibling) => sibling.tagName === current.tagName);
const index = siblings.indexOf(current) + 1;
parts.unshift(siblings.length > 1 ? tag + ':nth-of-type(' + index + ')' : tag);
current = parent;
}
return parts.length > 0 ? parts.join(' > ') : element.tagName.toLowerCase();
}
return JSON.stringify(Array.from(document.querySelectorAll(CANDIDATE_SELECTOR))
.filter(isVisible)
.slice(0, MAX_ELEMENTS)
.map((element) => ({
role: roleFor(element),
tagName: element.tagName.toLowerCase(),
text: textFor(element),
selector: selectorFor(element),
attributes: {
...(element.id ? { id: element.id } : {}),
...(element.getAttribute('name') ? { name: element.getAttribute('name') } : {}),
...(element.getAttribute('type') ? { type: element.getAttribute('type') } : {}),
...(element.getAttribute('href') ? { href: element.getAttribute('href') } : {}),
...(element.getAttribute('aria-label') ? { 'aria-label': element.getAttribute('aria-label') } : {})
}
})));
})()`;
@@ -4,11 +4,12 @@ import {
handleBrowserWindowOpenRequest, handleBrowserWindowOpenRequest,
isAllowedBrowserWebviewUrl, isAllowedBrowserWebviewUrl,
} from "./window-open.js"; } from "./window-open.js";
import { PaseoBrowserWebviewRegistry, type BrowserWorkspaceRegistration } from "./registry.js";
export { BROWSER_NEW_TAB_REQUEST_EVENT, handleBrowserWindowOpenRequest }; export { BROWSER_NEW_TAB_REQUEST_EVENT, handleBrowserWindowOpenRequest };
export type { BrowserWorkspaceRegistration };
const browserIdsByWebContentsId = new Map<number, string>(); const browserRegistry = new PaseoBrowserWebviewRegistry();
let workspaceActiveBrowserId: string | null = null;
function getBrowserIdFromWebviewPartition(partition: string | undefined): string | null { function getBrowserIdFromWebviewPartition(partition: string | undefined): string | null {
const prefix = "persist:paseo-browser-"; const prefix = "persist:paseo-browser-";
@@ -30,16 +31,15 @@ export function readBrowserIdFromWebviewAttach(input: {
} }
export function listRegisteredPaseoBrowserIds(): string[] { export function listRegisteredPaseoBrowserIds(): string[] {
return Array.from(new Set(browserIdsByWebContentsId.values())).sort(); return browserRegistry
.listBrowserIds()
.filter((browserId) => getPaseoBrowserWebContents(browserId));
} }
export function registerPaseoBrowserWebContents(contents: WebContents, browserId: string): void { export function registerPaseoBrowserWebContents(contents: WebContents, browserId: string): void {
browserIdsByWebContentsId.set(contents.id, browserId); browserRegistry.registerWebContents({ webContentsId: contents.id, browserId });
contents.once("destroyed", () => { contents.once("destroyed", () => {
browserIdsByWebContentsId.delete(contents.id); browserRegistry.unregisterWebContents(contents.id);
if (workspaceActiveBrowserId === browserId) {
workspaceActiveBrowserId = null;
}
}); });
} }
@@ -47,29 +47,71 @@ export function getPaseoBrowserIdForWebContents(contents: WebContents | null): s
if (!contents || contents.isDestroyed()) { if (!contents || contents.isDestroyed()) {
return null; return null;
} }
return browserIdsByWebContentsId.get(contents.id) ?? null; return browserRegistry.getBrowserIdForWebContents(contents.id);
} }
export function setWorkspaceActivePaseoBrowserId(browserId: string | null): void { export function registerPaseoBrowserWorkspace(input: BrowserWorkspaceRegistration): void {
workspaceActiveBrowserId = browserId; browserRegistry.registerWorkspace(input);
}
export function getPaseoBrowserWorkspaceId(browserId: string): string | null {
return browserRegistry.getWorkspaceId(browserId);
}
export function listRegisteredPaseoBrowserIdsForWorkspace(workspaceId: string): string[] {
return browserRegistry
.listBrowserIdsForWorkspace(workspaceId)
.filter((browserId) => getPaseoBrowserWebContents(browserId));
}
export function setWorkspaceActivePaseoBrowserId(input: {
workspaceId: string;
browserId: string | null;
}): void {
browserRegistry.setWorkspaceActiveBrowser(input);
}
export function getWorkspaceActivePaseoBrowserId(workspaceId: string): string | null {
return browserRegistry.getWorkspaceActiveBrowserId(workspaceId);
}
export function setAgentActivePaseoBrowserId(input: {
agentId: string;
browserId: string | null;
}): void {
browserRegistry.setAgentActiveBrowser(input);
}
export function getAgentActivePaseoBrowserId(agentId: string): string | null {
return browserRegistry.getAgentActiveBrowserId(agentId);
} }
export function getPaseoBrowserWebContents(browserId: string): WebContents | null { export function getPaseoBrowserWebContents(browserId: string): WebContents | null {
for (const [contentsId, registeredBrowserId] of browserIdsByWebContentsId) { const contentsId = browserRegistry.getWebContentsIdForBrowser(browserId);
if (registeredBrowserId !== browserId) continue; if (contentsId === null) {
const contents = allWebContents.fromId(contentsId); return null;
if (contents && !contents.isDestroyed()) {
return contents;
}
} }
const contents = allWebContents.fromId(contentsId);
if (contents && !contents.isDestroyed()) {
return contents;
}
browserRegistry.unregisterWebContents(contentsId);
return null; return null;
} }
export function getWorkspaceActivePaseoBrowserWebContents(): WebContents | null { export function getWorkspaceActivePaseoBrowserWebContents(workspaceId: string): WebContents | null {
if (!workspaceActiveBrowserId) { const activeBrowserId = getWorkspaceActivePaseoBrowserId(workspaceId);
return null; return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null;
} }
return getPaseoBrowserWebContents(workspaceActiveBrowserId);
export function getAgentActivePaseoBrowserWebContents(agentId: string): WebContents | null {
const activeBrowserId = getAgentActivePaseoBrowserId(agentId);
return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null;
}
export function getMostRecentWorkspaceActivePaseoBrowserWebContents(): WebContents | null {
const browserId = browserRegistry.getMostRecentWorkspaceActiveBrowserId();
return browserId ? getPaseoBrowserWebContents(browserId) : null;
} }
function preventUnsafeBrowserWebviewNavigation( function preventUnsafeBrowserWebviewNavigation(
@@ -0,0 +1,31 @@
import { describe, expect, it } from "vitest";
import { PaseoBrowserWebviewRegistry } from "./registry.js";
describe("PaseoBrowserWebviewRegistry", () => {
it("keeps one authoritative webContents target per browserId", () => {
const registry = new PaseoBrowserWebviewRegistry();
registry.registerWebContents({ webContentsId: 1, browserId: "browser-a" });
registry.registerWorkspace({ browserId: "browser-a", workspaceId: "workspace-a" });
registry.setWorkspaceActiveBrowser({ workspaceId: "workspace-a", browserId: "browser-a" });
registry.setAgentActiveBrowser({ agentId: "agent-a", browserId: "browser-a" });
registry.registerWebContents({ webContentsId: 2, browserId: "browser-a" });
expect(registry.getBrowserIdForWebContents(1)).toBeNull();
expect(registry.getBrowserIdForWebContents(2)).toBe("browser-a");
expect(registry.getWebContentsIdForBrowser("browser-a")).toBe(2);
expect(registry.getWorkspaceId("browser-a")).toBe("workspace-a");
expect(registry.getWorkspaceActiveBrowserId("workspace-a")).toBe("browser-a");
expect(registry.getAgentActiveBrowserId("agent-a")).toBe("browser-a");
});
it("ignores stale destroy events after a duplicate browserId moved", () => {
const registry = new PaseoBrowserWebviewRegistry();
registry.registerWebContents({ webContentsId: 1, browserId: "browser-a" });
registry.registerWebContents({ webContentsId: 2, browserId: "browser-a" });
registry.unregisterWebContents(1);
expect(registry.getWebContentsIdForBrowser("browser-a")).toBe(2);
});
});
@@ -0,0 +1,108 @@
export interface BrowserWorkspaceRegistration {
browserId: string;
workspaceId: string;
}
export class PaseoBrowserWebviewRegistry {
private readonly browserIdsByWebContentsId = new Map<number, string>();
private readonly webContentsIdsByBrowserId = new Map<string, number>();
private readonly workspaceIdsByBrowserId = new Map<string, string>();
private readonly activeBrowserIdsByWorkspaceId = new Map<string, string>();
private readonly activeBrowserIdsByAgentId = new Map<string, string>();
public registerWebContents(input: { webContentsId: number; browserId: string }): void {
const previousWebContentsId = this.webContentsIdsByBrowserId.get(input.browserId) ?? null;
if (previousWebContentsId !== null && previousWebContentsId !== input.webContentsId) {
this.browserIdsByWebContentsId.delete(previousWebContentsId);
}
this.browserIdsByWebContentsId.set(input.webContentsId, input.browserId);
this.webContentsIdsByBrowserId.set(input.browserId, input.webContentsId);
}
public unregisterWebContents(webContentsId: number): void {
const browserId = this.browserIdsByWebContentsId.get(webContentsId) ?? null;
if (!browserId) {
return;
}
this.browserIdsByWebContentsId.delete(webContentsId);
if (this.webContentsIdsByBrowserId.get(browserId) !== webContentsId) {
return;
}
this.webContentsIdsByBrowserId.delete(browserId);
this.workspaceIdsByBrowserId.delete(browserId);
this.deleteActiveBrowserReferences(browserId);
}
public getBrowserIdForWebContents(webContentsId: number): string | null {
return this.browserIdsByWebContentsId.get(webContentsId) ?? null;
}
public getWebContentsIdForBrowser(browserId: string): number | null {
return this.webContentsIdsByBrowserId.get(browserId) ?? null;
}
public listBrowserIds(): string[] {
return Array.from(this.webContentsIdsByBrowserId.keys()).sort();
}
public registerWorkspace(input: BrowserWorkspaceRegistration): void {
this.workspaceIdsByBrowserId.set(input.browserId, input.workspaceId);
}
public getWorkspaceId(browserId: string): string | null {
return this.workspaceIdsByBrowserId.get(browserId) ?? null;
}
public listBrowserIdsForWorkspace(workspaceId: string): string[] {
return this.listBrowserIds().filter(
(browserId) => this.workspaceIdsByBrowserId.get(browserId) === workspaceId,
);
}
public setWorkspaceActiveBrowser(input: { workspaceId: string; browserId: string | null }): void {
if (input.browserId) {
this.workspaceIdsByBrowserId.set(input.browserId, input.workspaceId);
this.activeBrowserIdsByWorkspaceId.delete(input.workspaceId);
this.activeBrowserIdsByWorkspaceId.set(input.workspaceId, input.browserId);
return;
}
this.activeBrowserIdsByWorkspaceId.delete(input.workspaceId);
}
public getWorkspaceActiveBrowserId(workspaceId: string): string | null {
return this.activeBrowserIdsByWorkspaceId.get(workspaceId) ?? null;
}
public getMostRecentWorkspaceActiveBrowserId(): string | null {
return Array.from(this.activeBrowserIdsByWorkspaceId.values()).at(-1) ?? null;
}
public setAgentActiveBrowser(input: { agentId: string; browserId: string | null }): void {
if (input.browserId) {
this.activeBrowserIdsByAgentId.delete(input.agentId);
this.activeBrowserIdsByAgentId.set(input.agentId, input.browserId);
return;
}
this.activeBrowserIdsByAgentId.delete(input.agentId);
}
public getAgentActiveBrowserId(agentId: string): string | null {
return this.activeBrowserIdsByAgentId.get(agentId) ?? null;
}
private deleteActiveBrowserReferences(browserId: string): void {
for (const [workspaceId, activeBrowserId] of this.activeBrowserIdsByWorkspaceId) {
if (activeBrowserId === browserId) {
this.activeBrowserIdsByWorkspaceId.delete(workspaceId);
}
}
for (const [agentId, activeBrowserId] of this.activeBrowserIdsByAgentId) {
if (activeBrowserId === browserId) {
this.activeBrowserIdsByAgentId.delete(agentId);
}
}
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
import { app, Menu, BrowserWindow, ipcMain } from "electron"; import { app, Menu, BrowserWindow, ipcMain } from "electron";
import { getWorkspaceActivePaseoBrowserWebContents } from "./browser-webviews/index.js"; import { getMostRecentWorkspaceActivePaseoBrowserWebContents } from "./browser-webviews/index.js";
interface ShowContextMenuInput { interface ShowContextMenuInput {
kind?: "terminal"; kind?: "terminal";
@@ -20,7 +20,7 @@ function withBrowserWindow(
} }
function getReloadTargetBrowserWebContents(): Electron.WebContents | null { function getReloadTargetBrowserWebContents(): Electron.WebContents | null {
return getWorkspaceActivePaseoBrowserWebContents(); return getMostRecentWorkspaceActivePaseoBrowserWebContents();
} }
function reloadFocusedContentsOrWindow(win: BrowserWindow, options?: { ignoreCache?: boolean }) { function reloadFocusedContentsOrWindow(win: BrowserWindow, options?: { ignoreCache?: boolean }) {
+67 -2
View File
@@ -53,7 +53,9 @@ import {
listRegisteredPaseoBrowserIds, listRegisteredPaseoBrowserIds,
readBrowserIdFromWebviewAttach, readBrowserIdFromWebviewAttach,
registerBrowserWebviewNavigationGuards, registerBrowserWebviewNavigationGuards,
registerPaseoBrowserWorkspace,
registerPaseoBrowserWebContents, registerPaseoBrowserWebContents,
setAgentActivePaseoBrowserId,
setWorkspaceActivePaseoBrowserId, setWorkspaceActivePaseoBrowserId,
} from "./features/browser-webviews/index.js"; } from "./features/browser-webviews/index.js";
import { parseOpenProjectPathFromArgv } from "./open-project-routing.js"; import { parseOpenProjectPathFromArgv } from "./open-project-routing.js";
@@ -70,6 +72,7 @@ import {
} from "./daemon/quit-lifecycle.js"; } from "./daemon/quit-lifecycle.js";
import { runDesktopStartup } from "./desktop-startup.js"; import { runDesktopStartup } from "./desktop-startup.js";
import { autoUpdateInstalledSkills } from "./integrations/skills/index.js"; import { autoUpdateInstalledSkills } from "./integrations/skills/index.js";
import { registerBrowserAutomationIpc } from "./features/browser-automation/ipc.js";
const DEV_SERVER_URL = process.env.EXPO_DEV_URL ?? "http://localhost:8081"; const DEV_SERVER_URL = process.env.EXPO_DEV_URL ?? "http://localhost:8081";
const APP_SCHEME = "paseo"; const APP_SCHEME = "paseo";
@@ -109,6 +112,50 @@ const DESKTOP_SMOKE_ENV = "PASEO_DESKTOP_SMOKE";
const DESKTOP_SMOKE_STOP_REQUEST = "paseo-smoke-stop"; const DESKTOP_SMOKE_STOP_REQUEST = "paseo-smoke-stop";
app.setName(APP_NAME); app.setName(APP_NAME);
function readBrowserWorkspaceInput(
input: unknown,
): { browserId: string; workspaceId: string } | null {
if (typeof input !== "object" || input === null || Array.isArray(input)) {
return null;
}
const record = input as Record<string, unknown>;
if (typeof record.browserId !== "string" || record.browserId.trim().length === 0) {
return null;
}
if (typeof record.workspaceId !== "string" || record.workspaceId.trim().length === 0) {
return null;
}
return { browserId: record.browserId.trim(), workspaceId: record.workspaceId.trim() };
}
function readActiveBrowserInput(
input: unknown,
): { workspaceId: string; browserId: string | null } | null {
if (typeof input !== "object" || input === null || Array.isArray(input)) {
return null;
}
const record = input as Record<string, unknown>;
if (typeof record.workspaceId !== "string" || record.workspaceId.trim().length === 0) {
return null;
}
const browserId = typeof record.browserId === "string" ? record.browserId.trim() : null;
return { workspaceId: record.workspaceId.trim(), browserId: browserId || null };
}
function readAgentActiveBrowserInput(
input: unknown,
): { agentId: string; browserId: string | null } | null {
if (typeof input !== "object" || input === null || Array.isArray(input)) {
return null;
}
const record = input as Record<string, unknown>;
if (typeof record.agentId !== "string" || record.agentId.trim().length === 0) {
return null;
}
const browserId = typeof record.browserId === "string" ? record.browserId.trim() : null;
return { agentId: record.agentId.trim(), browserId: browserId || null };
}
const pendingBrowserWebviewIds: string[] = []; const pendingBrowserWebviewIds: string[] = [];
function isBrowserRefreshInput(input: Electron.Input): boolean { function isBrowserRefreshInput(input: Electron.Input): boolean {
@@ -254,8 +301,25 @@ ipcMain.handle("paseo:get-pending-open-project", (event) => {
return result; return result;
}); });
ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, browserId: unknown) => { ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: unknown) => {
setWorkspaceActivePaseoBrowserId(typeof browserId === "string" ? browserId : null); const input = readBrowserWorkspaceInput(rawInput);
if (input) {
registerPaseoBrowserWorkspace(input);
}
});
ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput: unknown) => {
const input = readActiveBrowserInput(rawInput);
if (input) {
setWorkspaceActivePaseoBrowserId(input);
}
});
ipcMain.handle("paseo:browser:set-agent-active-browser", (_event, rawInput: unknown) => {
const input = readAgentActiveBrowserInput(rawInput);
if (input) {
setAgentActivePaseoBrowserId(input);
}
}); });
ipcMain.handle("paseo:browser:open-devtools", (_event, browserId: unknown) => { ipcMain.handle("paseo:browser:open-devtools", (_event, browserId: unknown) => {
@@ -696,6 +760,7 @@ async function bootstrap(): Promise<void> {
registerNotificationHandlers(); registerNotificationHandlers();
registerOpenerHandlers(); registerOpenerHandlers();
registerEditorTargetHandlers(); registerEditorTargetHandlers();
registerBrowserAutomationIpc();
// In-app "Open in new window": opens a window that lands on the given project // In-app "Open in new window": opens a window that lands on the given project
// via the same open-project flow as a CLI launch (no move, no ownership). // via the same open-project flow as a CLI launch (no move, no ownership).
+8 -2
View File
@@ -74,11 +74,17 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
ipcRenderer.invoke("paseo:menu:showContextMenu", input), ipcRenderer.invoke("paseo:menu:showContextMenu", input),
}, },
browser: { browser: {
setWorkspaceActiveBrowser: (browserId: string | null) => registerWorkspaceBrowser: (input: { browserId: string; workspaceId: string }) =>
ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", browserId), ipcRenderer.invoke("paseo:browser:register-workspace-browser", input),
setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) =>
ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input),
setAgentActiveBrowser: (input: { agentId: string; browserId: string | null }) =>
ipcRenderer.invoke("paseo:browser:set-agent-active-browser", input),
openDevTools: (browserId: string) => openDevTools: (browserId: string) =>
ipcRenderer.invoke("paseo:browser:open-devtools", browserId), ipcRenderer.invoke("paseo:browser:open-devtools", browserId),
clearPartition: (browserId: string) => clearPartition: (browserId: string) =>
ipcRenderer.invoke("paseo:browser:clear-partition", browserId), ipcRenderer.invoke("paseo:browser:clear-partition", browserId),
executeAutomationCommand: (request: Record<string, unknown>) =>
ipcRenderer.invoke("paseo:browser:execute-automation-command", request),
}, },
}); });
@@ -0,0 +1,750 @@
import { describe, expect, test } from "vitest";
import {
BrowserAutomationExecuteRequestSchema,
BrowserAutomationExecuteResponseSchema,
} from "./rpc-schemas.js";
describe("browser automation execute RPC schemas", () => {
test("rejects navigate and download commands for non-http URLs", () => {
for (const command of ["navigate", "download"] as const) {
expect(() =>
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: `req-${command}`,
command: {
command,
args: { url: "file:///tmp/secret.txt" },
},
}),
).toThrow();
}
});
test("parses list tabs requests with top-level correlation and typed command args", () => {
const parsed = BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: {
command: "list_tabs",
args: { workspaceId: "workspace-1" },
},
});
expect(parsed).toEqual({
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: {
command: "list_tabs",
args: { workspaceId: "workspace-1" },
},
});
});
test("parses new tab requests and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-new-tab",
workspaceId: "workspace-1",
command: {
command: "new_tab",
args: { workspaceId: "workspace-1", url: "https://example.com" },
},
}).command,
).toEqual({
command: "new_tab",
args: { workspaceId: "workspace-1", url: "https://example.com" },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-new-tab",
ok: true,
result: {
command: "new_tab",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
},
},
}).payload,
).toEqual({
requestId: "req-new-tab",
ok: true,
result: {
command: "new_tab",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
},
});
});
test("parses page info responses with result data under payload", () => {
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: true,
result: {
command: "page_info",
tab: {
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
},
},
},
});
expect(parsed.payload).toEqual({
requestId: "req-1",
ok: true,
result: {
command: "page_info",
tab: {
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
isActive: false,
isLoading: false,
},
},
});
});
test("parses snapshot requests and ref-bearing snapshot responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-snapshot",
workspaceId: "workspace-1",
command: {
command: "snapshot",
args: { workspaceId: "workspace-1", browserId: "browser-1" },
},
}),
).toEqual({
type: "browser.automation.execute.request",
requestId: "req-snapshot",
workspaceId: "workspace-1",
command: {
command: "snapshot",
args: { workspaceId: "workspace-1", browserId: "browser-1" },
},
});
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-snapshot",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name", type: "text" },
},
],
},
},
});
expect(parsed.payload).toEqual({
requestId: "req-snapshot",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name", type: "text" },
},
],
},
});
});
test("parses click and fill ref commands", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-click",
command: {
command: "click",
args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" },
},
}).command,
).toEqual({
command: "click",
args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" },
});
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-fill",
command: {
command: "fill",
args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" },
},
}).command,
).toEqual({
command: "fill",
args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-fill",
ok: true,
result: { command: "fill", browserId: "browser-1", ref: "@e2" },
},
}).payload,
).toEqual({
requestId: "req-fill",
ok: true,
result: { command: "fill", browserId: "browser-1", ref: "@e2" },
});
});
test("parses wait text commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-wait",
command: {
command: "wait",
args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 },
},
}).command,
).toEqual({
command: "wait",
args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-wait",
ok: true,
result: { command: "wait", browserId: "browser-1", matched: "text" },
},
}).payload,
).toEqual({
requestId: "req-wait",
ok: true,
result: { command: "wait", browserId: "browser-1", matched: "text" },
});
});
test("parses set background commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-bg",
command: {
command: "set_background",
args: { workspaceId: "workspace-1", color: "red" },
},
}).command,
).toEqual({
command: "set_background",
args: { workspaceId: "workspace-1", color: "red" },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-bg",
ok: true,
result: { command: "set_background", browserId: "browser-1", color: "red" },
},
}).payload,
).toEqual({
requestId: "req-bg",
ok: true,
result: { command: "set_background", browserId: "browser-1", color: "red" },
});
});
test("parses type and keypress commands", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-type",
command: { command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } },
}).command,
).toEqual({ command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-keypress",
command: { command: "keypress", args: { browserId: "browser-1", key: "Enter" } },
}).command,
).toEqual({ command: "keypress", args: { browserId: "browser-1", key: "Enter" } });
});
test("parses navigation commands", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-nav",
command: {
command: "navigate",
args: { browserId: "browser-1", url: "https://example.com/next" },
},
}).command,
).toEqual({
command: "navigate",
args: { browserId: "browser-1", url: "https://example.com/next" },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-nav",
ok: true,
result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" },
},
}).payload,
).toEqual({
requestId: "req-nav",
ok: true,
result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" },
});
});
test("parses screenshot responses", () => {
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-shot",
ok: true,
result: {
command: "screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 100,
height: 50,
},
},
}).payload,
).toEqual({
requestId: "req-shot",
ok: true,
result: {
command: "screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 100,
height: 50,
},
});
});
test("parses form control commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-focus",
command: { command: "focus", args: { browserId: "browser-1", ref: "@e1" } },
}).command,
).toEqual({ command: "focus", args: { browserId: "browser-1", ref: "@e1" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-clear",
command: { command: "clear", args: { browserId: "browser-1", ref: "@e1" } },
}).command,
).toEqual({ command: "clear", args: { browserId: "browser-1", ref: "@e1" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-check",
command: { command: "check", args: { browserId: "browser-1", ref: "@e2" } },
}).command,
).toEqual({ command: "check", args: { browserId: "browser-1", ref: "@e2", checked: true } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-select",
command: { command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } },
}).command,
).toEqual({ command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-hover",
command: { command: "hover", args: { browserId: "browser-1", ref: "@e4" } },
}).command,
).toEqual({ command: "hover", args: { browserId: "browser-1", ref: "@e4" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-drag",
command: {
command: "drag",
args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" },
},
}).command,
).toEqual({
command: "drag",
args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-select",
ok: true,
result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" },
},
}).payload,
).toEqual({
requestId: "req-select",
ok: true,
result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" },
});
});
test("parses browser log commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-logs",
command: { command: "logs", args: { browserId: "browser-1" } },
}).command,
).toEqual({ command: "logs", args: { browserId: "browser-1", maxEntries: 50 } });
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-logs",
ok: true,
result: {
command: "logs",
browserId: "browser-1",
console: [{ level: "info", message: "ready", timestamp: 10 }],
network: [
{
url: "https://example.com/app.js",
type: "script",
startTime: 1,
duration: 2,
},
],
},
},
}).payload,
).toEqual({
requestId: "req-logs",
ok: true,
result: {
command: "logs",
browserId: "browser-1",
console: [{ level: "info", message: "ready", timestamp: 10 }],
network: [
{
url: "https://example.com/app.js",
type: "script",
startTime: 1,
duration: 2,
},
],
},
});
});
test("parses browser storage commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-storage",
command: { command: "storage", args: { browserId: "browser-1" } },
}).command,
).toEqual({ command: "storage", args: { browserId: "browser-1" } });
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-storage",
ok: true,
result: {
command: "storage",
browserId: "browser-1",
url: "https://example.com",
cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
},
}).payload,
).toEqual({
requestId: "req-storage",
ok: true,
result: {
command: "storage",
browserId: "browser-1",
url: "https://example.com",
cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
});
});
test("parses browser environment commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-environment",
command: {
command: "environment",
args: {
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
}).command,
).toEqual({
command: "environment",
args: {
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-environment",
ok: true,
result: {
command: "environment",
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
}).payload,
).toEqual({
requestId: "req-environment",
ok: true,
result: {
command: "environment",
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
});
});
test("parses browser full-page screenshot and PDF commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-full-page",
command: { command: "full_page_screenshot", args: { browserId: "browser-1" } },
}).command,
).toEqual({ command: "full_page_screenshot", args: { browserId: "browser-1" } });
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-pdf",
command: { command: "pdf", args: { browserId: "browser-1" } },
}).command,
).toEqual({ command: "pdf", args: { browserId: "browser-1", printBackground: true } });
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-full-page",
ok: true,
result: {
command: "full_page_screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 390,
height: 1200,
},
},
}).payload.result,
).toEqual({
command: "full_page_screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 390,
height: 1200,
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-pdf",
ok: true,
result: {
command: "pdf",
browserId: "browser-1",
mimeType: "application/pdf",
dataBase64: "JVBERi0xLjQ=",
},
},
}).payload.result,
).toEqual({
command: "pdf",
browserId: "browser-1",
mimeType: "application/pdf",
dataBase64: "JVBERi0xLjQ=",
});
});
test("parses browser download and upload commands and responses", () => {
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-download",
command: {
command: "download",
args: { browserId: "browser-1", url: "https://example.com/file.txt" },
},
}).command,
).toEqual({
command: "download",
args: { browserId: "browser-1", url: "https://example.com/file.txt" },
});
expect(
BrowserAutomationExecuteRequestSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-upload",
command: {
command: "upload",
args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] },
},
}).command,
).toEqual({
command: "upload",
args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] },
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-download",
ok: true,
result: {
command: "download",
browserId: "browser-1",
url: "https://example.com/file.txt",
filePath: "/tmp/file.txt",
totalBytes: 5,
state: "completed",
},
},
}).payload.result,
).toEqual({
command: "download",
browserId: "browser-1",
url: "https://example.com/file.txt",
filePath: "/tmp/file.txt",
totalBytes: 5,
state: "completed",
});
expect(
BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-upload",
ok: true,
result: {
command: "upload",
browserId: "browser-1",
ref: "@e1",
filePaths: ["/tmp/file.txt"],
},
},
}).payload.result,
).toEqual({
command: "upload",
browserId: "browser-1",
ref: "@e1",
filePaths: ["/tmp/file.txt"],
});
});
test("parses stable model-actionable error responses", () => {
const parsed = BrowserAutomationExecuteResponseSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: false,
error: {
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
},
},
});
expect(parsed.payload).toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
retryable: false,
},
});
});
});
@@ -0,0 +1,599 @@
import { z } from "zod";
export const BrowserAutomationErrorCodeSchema = z.enum([
"browser_disabled",
"browser_no_desktop",
"browser_no_tab",
"browser_tab_not_found",
"browser_tab_closed",
"browser_timeout",
"browser_denied",
"browser_unsupported",
"browser_stale_ref",
"browser_unknown_error",
]);
const BrowserAutomationTabTargetSchema = z.object({
workspaceId: z.string().min(1).optional(),
browserId: z.string().min(1).optional(),
});
const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/);
const BrowserAutomationHttpUrlSchema = z
.string()
.url()
.refine((value) => {
const protocol = new URL(value).protocol;
return protocol === "http:" || protocol === "https:";
}, "URL must use http or https");
export const BrowserAutomationListTabsCommandSchema = z.object({
command: z.literal("list_tabs"),
args: z
.object({
workspaceId: z.string().min(1).optional(),
})
.default({}),
});
export const BrowserAutomationNewTabCommandSchema = z.object({
command: z.literal("new_tab"),
args: z
.object({
workspaceId: z.string().min(1).optional(),
url: BrowserAutomationHttpUrlSchema.optional(),
})
.default({}),
});
export const BrowserAutomationPageInfoCommandSchema = z.object({
command: z.literal("page_info"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationSnapshotCommandSchema = z.object({
command: z.literal("snapshot"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationClickCommandSchema = z.object({
command: z.literal("click"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationFillCommandSchema = z.object({
command: z.literal("fill"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
value: z.string(),
}),
});
export const BrowserAutomationWaitCommandSchema = z.object({
command: z.literal("wait"),
args: BrowserAutomationTabTargetSchema.extend({
text: z.string().min(1).optional(),
url: z.string().min(1).optional(),
timeoutMs: z.number().int().positive().max(30_000).optional(),
}),
});
export const BrowserAutomationTypeCommandSchema = z.object({
command: z.literal("type"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema.optional(),
text: z.string(),
}),
});
export const BrowserAutomationKeypressCommandSchema = z.object({
command: z.literal("keypress"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema.optional(),
key: z.string().min(1),
}),
});
export const BrowserAutomationNavigateCommandSchema = z.object({
command: z.literal("navigate"),
args: BrowserAutomationTabTargetSchema.extend({
url: BrowserAutomationHttpUrlSchema,
}),
});
export const BrowserAutomationBackCommandSchema = z.object({
command: z.literal("back"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationForwardCommandSchema = z.object({
command: z.literal("forward"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationReloadCommandSchema = z.object({
command: z.literal("reload"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationScreenshotCommandSchema = z.object({
command: z.literal("screenshot"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({
command: z.literal("full_page_screenshot"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationPdfCommandSchema = z.object({
command: z.literal("pdf"),
args: BrowserAutomationTabTargetSchema.extend({
landscape: z.boolean().optional(),
printBackground: z.boolean().default(true),
}).default({ printBackground: true }),
});
export const BrowserAutomationDownloadCommandSchema = z.object({
command: z.literal("download"),
args: BrowserAutomationTabTargetSchema.extend({
url: BrowserAutomationHttpUrlSchema,
fileName: z.string().min(1).optional(),
}),
});
export const BrowserAutomationUploadCommandSchema = z.object({
command: z.literal("upload"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
filePaths: z.array(z.string().min(1)).min(1),
}),
});
export const BrowserAutomationFocusCommandSchema = z.object({
command: z.literal("focus"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationClearCommandSchema = z.object({
command: z.literal("clear"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationCheckCommandSchema = z.object({
command: z.literal("check"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
checked: z.boolean().default(true),
}),
});
export const BrowserAutomationSelectCommandSchema = z.object({
command: z.literal("select"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
value: z.string(),
}),
});
export const BrowserAutomationHoverCommandSchema = z.object({
command: z.literal("hover"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationDragCommandSchema = z.object({
command: z.literal("drag"),
args: BrowserAutomationTabTargetSchema.extend({
sourceRef: BrowserAutomationRefSchema,
targetRef: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationLogsCommandSchema = z.object({
command: z.literal("logs"),
args: BrowserAutomationTabTargetSchema.extend({
maxEntries: z.number().int().positive().max(200).default(50),
}).default({ maxEntries: 50 }),
});
export const BrowserAutomationStorageCommandSchema = z.object({
command: z.literal("storage"),
args: BrowserAutomationTabTargetSchema.default({}),
});
const BrowserAutomationViewportInputSchema = z.object({
width: z.number().int().positive(),
height: z.number().int().positive(),
deviceScaleFactor: z.number().positive().optional(),
});
const BrowserAutomationGeolocationInputSchema = z.object({
latitude: z.number().min(-90).max(90),
longitude: z.number().min(-180).max(180),
accuracy: z.number().positive().optional(),
});
export const BrowserAutomationEnvironmentCommandSchema = z.object({
command: z.literal("environment"),
args: BrowserAutomationTabTargetSchema.extend({
viewport: BrowserAutomationViewportInputSchema.optional(),
geolocation: BrowserAutomationGeolocationInputSchema.optional(),
}).default({}),
});
export const BrowserAutomationSetBackgroundCommandSchema = z.object({
command: z.literal("set_background"),
args: BrowserAutomationTabTargetSchema.extend({
color: z.string().min(1),
}),
});
export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsCommandSchema,
BrowserAutomationNewTabCommandSchema,
BrowserAutomationPageInfoCommandSchema,
BrowserAutomationSnapshotCommandSchema,
BrowserAutomationClickCommandSchema,
BrowserAutomationFillCommandSchema,
BrowserAutomationWaitCommandSchema,
BrowserAutomationTypeCommandSchema,
BrowserAutomationKeypressCommandSchema,
BrowserAutomationNavigateCommandSchema,
BrowserAutomationBackCommandSchema,
BrowserAutomationForwardCommandSchema,
BrowserAutomationReloadCommandSchema,
BrowserAutomationScreenshotCommandSchema,
BrowserAutomationFullPageScreenshotCommandSchema,
BrowserAutomationPdfCommandSchema,
BrowserAutomationDownloadCommandSchema,
BrowserAutomationUploadCommandSchema,
BrowserAutomationFocusCommandSchema,
BrowserAutomationClearCommandSchema,
BrowserAutomationCheckCommandSchema,
BrowserAutomationSelectCommandSchema,
BrowserAutomationHoverCommandSchema,
BrowserAutomationDragCommandSchema,
BrowserAutomationLogsCommandSchema,
BrowserAutomationStorageCommandSchema,
BrowserAutomationEnvironmentCommandSchema,
BrowserAutomationSetBackgroundCommandSchema,
]);
export const BrowserAutomationTabInfoSchema = z.object({
browserId: z.string().min(1),
workspaceId: z.string().min(1).optional(),
url: z.string(),
title: z.string(),
isActive: z.boolean().default(false),
isLoading: z.boolean().default(false),
canGoBack: z.boolean().optional(),
canGoForward: z.boolean().optional(),
});
export const BrowserAutomationListTabsResultSchema = z.object({
command: z.literal("list_tabs"),
tabs: z.array(BrowserAutomationTabInfoSchema),
});
export const BrowserAutomationNewTabResultSchema = z.object({
command: z.literal("new_tab"),
browserId: z.string().min(1),
workspaceId: z.string().min(1),
url: z.string().min(1),
});
export const BrowserAutomationPageInfoResultSchema = z.object({
command: z.literal("page_info"),
tab: BrowserAutomationTabInfoSchema,
});
export const BrowserAutomationSnapshotElementSchema = z.object({
ref: z.string().regex(/^@e\d+$/),
role: z.string(),
tagName: z.string(),
text: z.string(),
selector: z.string(),
attributes: z.record(z.string(), z.string()).default({}),
});
export const BrowserAutomationSnapshotResultSchema = z.object({
command: z.literal("snapshot"),
browserId: z.string().min(1),
workspaceId: z.string().min(1).optional(),
url: z.string(),
title: z.string(),
elements: z.array(BrowserAutomationSnapshotElementSchema),
});
export const BrowserAutomationClickResultSchema = z.object({
command: z.literal("click"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationFillResultSchema = z.object({
command: z.literal("fill"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationWaitResultSchema = z.object({
command: z.literal("wait"),
browserId: z.string().min(1),
matched: z.enum(["text", "url"]),
});
export const BrowserAutomationTypeResultSchema = z.object({
command: z.literal("type"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
});
export const BrowserAutomationKeypressResultSchema = z.object({
command: z.literal("keypress"),
browserId: z.string().min(1),
key: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
});
export const BrowserAutomationNavigateResultSchema = z.object({
command: z.literal("navigate"),
browserId: z.string().min(1),
url: z.string().min(1),
});
export const BrowserAutomationBackResultSchema = z.object({
command: z.literal("back"),
browserId: z.string().min(1),
});
export const BrowserAutomationForwardResultSchema = z.object({
command: z.literal("forward"),
browserId: z.string().min(1),
});
export const BrowserAutomationReloadResultSchema = z.object({
command: z.literal("reload"),
browserId: z.string().min(1),
});
export const BrowserAutomationScreenshotResultSchema = z.object({
command: z.literal("screenshot"),
browserId: z.string().min(1),
mimeType: z.literal("image/png"),
dataBase64: z.string().min(1),
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
});
export const BrowserAutomationFullPageScreenshotResultSchema = z.object({
command: z.literal("full_page_screenshot"),
browserId: z.string().min(1),
mimeType: z.literal("image/png"),
dataBase64: z.string().min(1),
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
});
export const BrowserAutomationPdfResultSchema = z.object({
command: z.literal("pdf"),
browserId: z.string().min(1),
mimeType: z.literal("application/pdf"),
dataBase64: z.string().min(1),
});
export const BrowserAutomationDownloadResultSchema = z.object({
command: z.literal("download"),
browserId: z.string().min(1),
url: z.string().min(1),
filePath: z.string().min(1),
totalBytes: z.number().int().nonnegative().optional(),
state: z.string().min(1),
});
export const BrowserAutomationUploadResultSchema = z.object({
command: z.literal("upload"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
filePaths: z.array(z.string().min(1)).min(1),
});
export const BrowserAutomationFocusResultSchema = z.object({
command: z.literal("focus"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationClearResultSchema = z.object({
command: z.literal("clear"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationCheckResultSchema = z.object({
command: z.literal("check"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
checked: z.boolean(),
});
export const BrowserAutomationSelectResultSchema = z.object({
command: z.literal("select"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
value: z.string(),
});
export const BrowserAutomationHoverResultSchema = z.object({
command: z.literal("hover"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationDragResultSchema = z.object({
command: z.literal("drag"),
browserId: z.string().min(1),
sourceRef: BrowserAutomationRefSchema,
targetRef: BrowserAutomationRefSchema,
});
export const BrowserAutomationConsoleLogEntrySchema = z.object({
level: z.string(),
message: z.string(),
source: z.string().optional(),
line: z.number().int().optional(),
timestamp: z.number(),
});
export const BrowserAutomationNetworkLogEntrySchema = z.object({
url: z.string(),
method: z.string().optional(),
status: z.number().int().optional(),
type: z.string().optional(),
startTime: z.number(),
duration: z.number(),
transferSize: z.number().optional(),
});
export const BrowserAutomationLogsResultSchema = z.object({
command: z.literal("logs"),
browserId: z.string().min(1),
console: z.array(BrowserAutomationConsoleLogEntrySchema),
network: z.array(BrowserAutomationNetworkLogEntrySchema),
});
export const BrowserAutomationCookieEntrySchema = z.object({
name: z.string(),
value: z.string(),
domain: z.string().optional(),
path: z.string().optional(),
secure: z.boolean().optional(),
httpOnly: z.boolean().optional(),
expirationDate: z.number().optional(),
});
export const BrowserAutomationStorageEntrySchema = z.object({
key: z.string(),
value: z.string(),
});
export const BrowserAutomationStorageResultSchema = z.object({
command: z.literal("storage"),
browserId: z.string().min(1),
url: z.string(),
cookies: z.array(BrowserAutomationCookieEntrySchema),
localStorage: z.array(BrowserAutomationStorageEntrySchema),
sessionStorage: z.array(BrowserAutomationStorageEntrySchema),
});
export const BrowserAutomationViewportResultSchema = z.object({
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
deviceScaleFactor: z.number().positive(),
});
export const BrowserAutomationGeolocationResultSchema = z.object({
latitude: z.number(),
longitude: z.number(),
accuracy: z.number(),
});
export const BrowserAutomationEnvironmentResultSchema = z.object({
command: z.literal("environment"),
browserId: z.string().min(1),
viewport: BrowserAutomationViewportResultSchema,
geolocation: BrowserAutomationGeolocationResultSchema.optional(),
});
export const BrowserAutomationSetBackgroundResultSchema = z.object({
command: z.literal("set_background"),
browserId: z.string().min(1),
color: z.string().min(1),
});
export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsResultSchema,
BrowserAutomationNewTabResultSchema,
BrowserAutomationPageInfoResultSchema,
BrowserAutomationSnapshotResultSchema,
BrowserAutomationClickResultSchema,
BrowserAutomationFillResultSchema,
BrowserAutomationWaitResultSchema,
BrowserAutomationTypeResultSchema,
BrowserAutomationKeypressResultSchema,
BrowserAutomationNavigateResultSchema,
BrowserAutomationBackResultSchema,
BrowserAutomationForwardResultSchema,
BrowserAutomationReloadResultSchema,
BrowserAutomationScreenshotResultSchema,
BrowserAutomationFullPageScreenshotResultSchema,
BrowserAutomationPdfResultSchema,
BrowserAutomationDownloadResultSchema,
BrowserAutomationUploadResultSchema,
BrowserAutomationFocusResultSchema,
BrowserAutomationClearResultSchema,
BrowserAutomationCheckResultSchema,
BrowserAutomationSelectResultSchema,
BrowserAutomationHoverResultSchema,
BrowserAutomationDragResultSchema,
BrowserAutomationLogsResultSchema,
BrowserAutomationStorageResultSchema,
BrowserAutomationEnvironmentResultSchema,
BrowserAutomationSetBackgroundResultSchema,
]);
export const BrowserAutomationErrorSchema = z.object({
code: BrowserAutomationErrorCodeSchema,
message: z.string().min(1),
retryable: z.boolean().default(false),
});
export const BrowserAutomationExecuteRequestSchema = z.object({
type: z.literal("browser.automation.execute.request"),
requestId: z.string().min(1),
agentId: z.string().min(1).optional(),
cwd: z.string().min(1).optional(),
workspaceId: z.string().min(1).optional(),
browserId: z.string().min(1).optional(),
command: BrowserAutomationCommandSchema,
});
export const BrowserAutomationExecuteResponseSchema = z.object({
type: z.literal("browser.automation.execute.response"),
payload: z.discriminatedUnion("ok", [
z.object({
requestId: z.string().min(1),
ok: z.literal(true),
result: BrowserAutomationResultSchema,
}),
z.object({
requestId: z.string().min(1),
ok: z.literal(false),
error: BrowserAutomationErrorSchema,
}),
]),
});
export type BrowserAutomationErrorCode = z.infer<typeof BrowserAutomationErrorCodeSchema>;
export type BrowserAutomationCommand = z.infer<typeof BrowserAutomationCommandSchema>;
export type BrowserAutomationResult = z.infer<typeof BrowserAutomationResultSchema>;
export type BrowserAutomationConsoleLogEntry = z.infer<
typeof BrowserAutomationConsoleLogEntrySchema
>;
export type BrowserAutomationNetworkLogEntry = z.infer<
typeof BrowserAutomationNetworkLogEntrySchema
>;
export type BrowserAutomationCookieEntry = z.infer<typeof BrowserAutomationCookieEntrySchema>;
export type BrowserAutomationStorageEntry = z.infer<typeof BrowserAutomationStorageEntrySchema>;
export type BrowserAutomationExecuteRequest = z.infer<typeof BrowserAutomationExecuteRequestSchema>;
export type BrowserAutomationExecuteResponse = z.infer<
typeof BrowserAutomationExecuteResponseSchema
>;
@@ -11,6 +11,7 @@ export const CLIENT_CAPS = {
// Old clients use a strict TerminalState schema and would reject the extra fields. // Old clients use a strict TerminalState schema and would reject the extra fields.
// Drop the gate (always send the flags) when floor >= v0.1.88. // Drop the gate (always send the flags) when floor >= v0.1.88.
terminalReflowableSnapshot: "terminal_reflowable_snapshot", terminalReflowableSnapshot: "terminal_reflowable_snapshot",
desktopBrowserAutomation: "desktop_browser_automation",
} as const; } as const;
export type ClientCapability = (typeof CLIENT_CAPS)[keyof typeof CLIENT_CAPS]; export type ClientCapability = (typeof CLIENT_CAPS)[keyof typeof CLIENT_CAPS];
@@ -0,0 +1,74 @@
import { describe, expect, test } from "vitest";
import { CLIENT_CAPS } from "./client-capabilities.js";
import {
MutableDaemonConfigPatchSchema,
MutableDaemonConfigSchema,
SessionInboundMessageSchema,
SessionOutboundMessageSchema,
WSHelloMessageSchema,
} from "./messages.js";
describe("browser automation protocol integration", () => {
test("desktop automation capability parses in hello without narrowing old clients", () => {
expect(
WSHelloMessageSchema.parse({
type: "hello",
clientId: "client-1",
clientType: "mobile",
protocolVersion: 1,
capabilities: {
[CLIENT_CAPS.desktopBrowserAutomation]: true,
},
}).capabilities,
).toMatchObject({
[CLIENT_CAPS.desktopBrowserAutomation]: true,
});
expect(
WSHelloMessageSchema.parse({
type: "hello",
clientId: "old-client",
clientType: "mobile",
protocolVersion: 1,
}).capabilities,
).toBeUndefined();
});
test("daemon to desktop execute request is an outbound session message", () => {
const parsed = SessionOutboundMessageSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-1",
command: { command: "page_info", args: { browserId: "browser-1" } },
});
expect(parsed.type).toBe("browser.automation.execute.request");
});
test("desktop to daemon execute response is an inbound session message", () => {
const parsed = SessionInboundMessageSchema.parse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: true,
result: { command: "list_tabs", tabs: [] },
},
});
expect(parsed.type).toBe("browser.automation.execute.response");
});
test("mutable daemon config defaults browser tools off and accepts opt-in patches", () => {
expect(
MutableDaemonConfigSchema.parse({
mcp: { injectIntoAgents: false },
}).browserTools,
).toEqual({ enabled: false });
expect(
MutableDaemonConfigPatchSchema.parse({
browserTools: { enabled: true },
}).browserTools,
).toEqual({ enabled: true });
});
});
+14
View File
@@ -53,6 +53,10 @@ import {
LoopLogsResponseSchema, LoopLogsResponseSchema,
LoopStopResponseSchema, LoopStopResponseSchema,
} from "@getpaseo/protocol/loop/rpc-schemas"; } from "@getpaseo/protocol/loop/rpc-schemas";
import {
BrowserAutomationExecuteRequestSchema,
BrowserAutomationExecuteResponseSchema,
} from "./browser-automation/rpc-schemas.js";
import { import {
PaseoConfigRawSchema, PaseoConfigRawSchema,
PaseoLifecycleCommandRawSchema, PaseoLifecycleCommandRawSchema,
@@ -129,6 +133,11 @@ export const TerminalProfileSchema = z
export type TerminalProfile = z.infer<typeof TerminalProfileSchema>; export type TerminalProfile = z.infer<typeof TerminalProfileSchema>;
const MutableBrowserToolsConfigSchema = z
.object({
enabled: z.boolean().default(false),
})
.passthrough();
export const MutableDaemonConfigSchema = z export const MutableDaemonConfigSchema = z
.object({ .object({
mcp: z mcp: z
@@ -136,6 +145,7 @@ export const MutableDaemonConfigSchema = z
injectIntoAgents: z.boolean(), injectIntoAgents: z.boolean(),
}) })
.passthrough(), .passthrough(),
browserTools: MutableBrowserToolsConfigSchema.default({ enabled: false }),
providers: z.record(z.string(), MutableDaemonProviderConfigSchema).default({}), providers: z.record(z.string(), MutableDaemonProviderConfigSchema).default({}),
metadataGeneration: MutableMetadataGenerationConfigSchema.default({ providers: [] }), metadataGeneration: MutableMetadataGenerationConfigSchema.default({ providers: [] }),
autoArchiveAfterMerge: z.boolean().default(false), autoArchiveAfterMerge: z.boolean().default(false),
@@ -148,6 +158,7 @@ export const MutableDaemonConfigSchema = z
export const MutableDaemonConfigPatchSchema = z export const MutableDaemonConfigPatchSchema = z
.object({ .object({
mcp: MutableDaemonConfigSchema.shape.mcp.partial().optional(), mcp: MutableDaemonConfigSchema.shape.mcp.partial().optional(),
browserTools: MutableBrowserToolsConfigSchema.partial().optional(),
providers: z providers: z
.record(z.string(), MutableDaemonProviderConfigSchema.partial().passthrough()) .record(z.string(), MutableDaemonProviderConfigSchema.partial().passthrough())
.optional(), .optional(),
@@ -2027,6 +2038,7 @@ export const CaptureTerminalRequestSchema = z.object({
}); });
export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
BrowserAutomationExecuteResponseSchema,
VoiceAudioChunkMessageSchema, VoiceAudioChunkMessageSchema,
AbortRequestMessageSchema, AbortRequestMessageSchema,
AudioPlayedMessageSchema, AudioPlayedMessageSchema,
@@ -4153,6 +4165,7 @@ export const DaemonUpdateProgressMessageSchema = z.object({
export type DaemonUpdateProgressMessage = z.infer<typeof DaemonUpdateProgressMessageSchema>; export type DaemonUpdateProgressMessage = z.infer<typeof DaemonUpdateProgressMessageSchema>;
export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
BrowserAutomationExecuteRequestSchema,
ActivityLogMessageSchema, ActivityLogMessageSchema,
AssistantChunkMessageSchema, AssistantChunkMessageSchema,
AudioOutputMessageSchema, AudioOutputMessageSchema,
@@ -4658,6 +4671,7 @@ export const WSHelloMessageSchema = z.object({
[CLIENT_CAPS.reasoningMergeEnum]: z.boolean().optional(), [CLIENT_CAPS.reasoningMergeEnum]: z.boolean().optional(),
[CLIENT_CAPS.customModeIcons]: z.boolean().optional(), [CLIENT_CAPS.customModeIcons]: z.boolean().optional(),
[CLIENT_CAPS.terminalReflowableSnapshot]: z.boolean().optional(), [CLIENT_CAPS.terminalReflowableSnapshot]: z.boolean().optional(),
[CLIENT_CAPS.desktopBrowserAutomation]: z.boolean().optional(),
}) })
.passthrough() .passthrough()
.optional(), .optional(),
+4 -1
View File
@@ -4,6 +4,8 @@ import net from "node:net";
import { spawn, type ChildProcess } from "node:child_process"; import { spawn, type ChildProcess } from "node:child_process";
import { createRequire } from "node:module"; import { createRequire } from "node:module";
import { Buffer } from "node:buffer"; import { Buffer } from "node:buffer";
import { dirname, resolve as resolvePath } from "node:path";
import { fileURLToPath } from "node:url";
import { import {
generateKeyPair, generateKeyPair,
exportPublicKey, exportPublicKey,
@@ -16,6 +18,7 @@ import {
const nodeMajor = Number((process.versions.node ?? "0").split(".")[0] ?? "0"); const nodeMajor = Number((process.versions.node ?? "0").split(".")[0] ?? "0");
const shouldRunRelayE2e = process.env.FORCE_RELAY_E2E === "1" || nodeMajor < 25; const shouldRunRelayE2e = process.env.FORCE_RELAY_E2E === "1" || nodeMajor < 25;
const wranglerCliPath = createRequire(import.meta.url).resolve("wrangler/bin/wrangler.js"); const wranglerCliPath = createRequire(import.meta.url).resolve("wrangler/bin/wrangler.js");
const relayPackageRoot = resolvePath(dirname(fileURLToPath(import.meta.url)), "..");
const STARTUP_HOOK_TIMEOUT_MS = 90_000; const STARTUP_HOOK_TIMEOUT_MS = 90_000;
const SHUTDOWN_TIMEOUT_MS = 10_000; const SHUTDOWN_TIMEOUT_MS = 10_000;
@@ -61,7 +64,7 @@ function spawnRelayDevServer(port: number): ChildProcess {
"--show-interactive-dev-session=false", "--show-interactive-dev-session=false",
], ],
{ {
cwd: process.cwd(), cwd: relayPackageRoot,
env: { ...process.env }, env: { ...process.env },
stdio: ["ignore", "pipe", "pipe"], stdio: ["ignore", "pipe", "pipe"],
detached: false, detached: false,
@@ -578,6 +578,91 @@ function createPaseoWorktreeForMcpTest(options: {
}; };
} }
describe("browser MCP tools", () => {
const logger = createTestLogger();
it("keeps browser tools registered when browser tools are disabled", async () => {
const { agentManager, agentStorage, spies } = createTestDeps();
spies.agentManager.getAgent.mockReturnValue({ id: "agent-1", cwd: REPO_CWD });
const execute = vi.fn().mockResolvedValue({
requestId: "req-browser-disabled",
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled.",
retryable: false,
},
});
const server = await createAgentMcpServer({
agentManager,
agentStorage,
providerSnapshotManager: createOpenCodeManager().manager,
browserToolsBroker: { execute } as never,
callerAgentId: "agent-1",
logger,
});
const tool = registeredTool(server, "browser_list_tabs");
const response = await tool.handler({});
expect(lookupTool(server, "browser_page_info")).not.toBeUndefined();
expect(execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: REPO_CWD,
workspaceId: REPO_CWD,
command: { command: "list_tabs", args: { workspaceId: REPO_CWD } },
});
expect(response.structuredContent).toEqual({
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled.",
retryable: false,
},
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD },
});
});
it("wires browser tools through the browser tools broker", async () => {
const { agentManager, agentStorage, spies } = createTestDeps();
spies.agentManager.getAgent.mockReturnValue({ id: "agent-1", cwd: REPO_CWD });
const execute = vi.fn().mockResolvedValue({
requestId: "req-browser-tabs",
ok: true,
result: { command: "list_tabs", tabs: [] },
});
const server = await createAgentMcpServer({
agentManager,
agentStorage,
providerSnapshotManager: createOpenCodeManager().manager,
browserToolsBroker: { execute } as never,
callerAgentId: "agent-1",
logger,
});
const tool = registeredTool(server, "browser_list_tabs");
const response = await tool.handler({});
expect(execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: REPO_CWD,
workspaceId: REPO_CWD,
command: { command: "list_tabs", args: { workspaceId: REPO_CWD } },
});
expect(response.content).toEqual([
{
type: "text",
text: "No Paseo browser tabs are open. Call browser_new_tab to create one, then use the returned browserId or omit browserId for the active tab.",
},
]);
expect(response.structuredContent).toEqual({
ok: true,
result: { command: "list_tabs", tabs: [] },
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD },
});
});
});
describe("terminal MCP tools", () => { describe("terminal MCP tools", () => {
const logger = createTestLogger(); const logger = createTestLogger();
@@ -280,7 +280,7 @@ describe("claude tool-call mapper", () => {
name: "Grep", name: "Grep",
input: { input: {
pattern: '\\\\\\"cli\\\\\\""', pattern: '\\\\\\"cli\\\\\\""',
path: "/Users/moboudra/dev/paseo/packages/desktop/src", path: "/workspaces/paseo/packages/desktop/src",
output_mode: "content", output_mode: "content",
"-n": true, "-n": true,
}, },
@@ -188,9 +188,9 @@ describe("opencode tool-call mapper", () => {
toolName: "read", toolName: "read",
callId: "opencode-read-xml", callId: "opencode-read-xml",
status: "completed", status: "completed",
input: { filePath: "/Users/moboudra/dev/paseo/docs/release.md" }, input: { filePath: "/workspaces/paseo/docs/release.md" },
output: [ output: [
"<path>/Users/moboudra/dev/paseo/docs/release.md</path>", "<path>/workspaces/paseo/docs/release.md</path>",
"<type>file</type>", "<type>file</type>",
"<content>", "<content>",
"1: # Release", "1: # Release",
@@ -203,7 +203,7 @@ describe("opencode tool-call mapper", () => {
expect(item.detail).toEqual({ expect(item.detail).toEqual({
type: "read", type: "read",
filePath: "/Users/moboudra/dev/paseo/docs/release.md", filePath: "/workspaces/paseo/docs/release.md",
content: [ content: [
"1: # Release", "1: # Release",
"2:", "2:",
@@ -282,8 +282,7 @@ describe("opencode tool-call mapper", () => {
callId: "opencode-write-success-text", callId: "opencode-write-success-text",
status: "completed", status: "completed",
input: { input: {
filePath: filePath: "/workspaces/paseo/worktrees/cold-ladybug/dummy.txt",
"/Users/moboudra/dev/paseo/.dev/paseo-home/worktrees/1luy0po7/cold-ladybug/dummy.txt",
content: "hello world\n", content: "hello world\n",
}, },
output: "Wrote file successfully.", output: "Wrote file successfully.",
@@ -292,8 +291,7 @@ describe("opencode tool-call mapper", () => {
expect(item.detail).toEqual({ expect(item.detail).toEqual({
type: "write", type: "write",
filePath: filePath: "/workspaces/paseo/worktrees/cold-ladybug/dummy.txt",
"/Users/moboudra/dev/paseo/.dev/paseo-home/worktrees/1luy0po7/cold-ladybug/dummy.txt",
content: "hello world\n", content: "hello world\n",
}); });
}); });
@@ -305,9 +303,9 @@ describe("opencode tool-call mapper", () => {
callId: "opencode-edit-camel", callId: "opencode-edit-camel",
status: "completed", status: "completed",
input: { input: {
filePath: "/Users/moboudra/dev/paseo/packages/website/src/data/agent-pages.ts", filePath: "/workspaces/paseo/packages/website/src/data/agent-pages.ts",
oldString: 'metaTitle: "Junie agent Mobile and Desktop App, Open Source"', oldString: 'metaTitle: "Agent page"',
newString: 'metaTitle: "Junie Agent Mobile and Desktop App, Open Source"', newString: 'metaTitle: "Updated agent page"',
}, },
output: "Edit applied successfully.", output: "Edit applied successfully.",
}), }),
@@ -315,9 +313,9 @@ describe("opencode tool-call mapper", () => {
expect(item.detail).toEqual({ expect(item.detail).toEqual({
type: "edit", type: "edit",
filePath: "/Users/moboudra/dev/paseo/packages/website/src/data/agent-pages.ts", filePath: "/workspaces/paseo/packages/website/src/data/agent-pages.ts",
oldString: 'metaTitle: "Junie agent Mobile and Desktop App, Open Source"', oldString: 'metaTitle: "Agent page"',
newString: 'metaTitle: "Junie Agent Mobile and Desktop App, Open Source"', newString: 'metaTitle: "Updated agent page"',
}); });
}); });
@@ -73,6 +73,8 @@ import {
type CreatePaseoWorktreeCommandInput, type CreatePaseoWorktreeCommandInput,
listPaseoWorktreesCommand, listPaseoWorktreesCommand,
} from "../../worktree/commands.js"; } from "../../worktree/commands.js";
import { registerBrowserTools } from "../../browser-tools/tools.js";
import type { BrowserToolsBroker } from "../../browser-tools/broker.js";
import type { import type {
PaseoToolCatalog, PaseoToolCatalog,
PaseoToolConfig, PaseoToolConfig,
@@ -102,6 +104,7 @@ export interface PaseoToolHostDependencies {
createPaseoWorktree?: CreatePaseoWorktreeWorkflowFn; createPaseoWorktree?: CreatePaseoWorktreeWorkflowFn;
// Mints a fresh directory workspace for a cwd and returns its id. // Mints a fresh directory workspace for a cwd and returns its id.
ensureWorkspaceForCreate?: (cwd: string) => Promise<string>; ensureWorkspaceForCreate?: (cwd: string) => Promise<string>;
browserToolsBroker?: BrowserToolsBroker | null;
paseoHome?: string; paseoHome?: string;
worktreesRoot?: string; worktreesRoot?: string;
/** /**
@@ -1002,6 +1005,15 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
return toCatalog(); return toCatalog();
} }
if (options.browserToolsBroker) {
registerBrowserTools({
registerTool,
broker: options.browserToolsBroker,
callerAgentId,
resolveCallerAgent,
});
}
registerTool( registerTool(
"create_agent", "create_agent",
{ {
+46 -22
View File
@@ -118,7 +118,9 @@ import { FileBackedChatService } from "./chat/chat-service.js";
import { CheckoutDiffManager } from "./checkout-diff-manager.js"; import { CheckoutDiffManager } from "./checkout-diff-manager.js";
import { LoopService } from "./loop-service.js"; import { LoopService } from "./loop-service.js";
import { ScheduleService } from "./schedule/service.js"; import { ScheduleService } from "./schedule/service.js";
import { DaemonConfigStore } from "./daemon-config-store.js"; import { DaemonConfigStore, type MutableDaemonConfig } from "./daemon-config-store.js";
import { BrowserToolsBroker } from "./browser-tools/broker.js";
import { DaemonConfigBrowserToolsPolicy } from "./browser-tools/policy.js";
import { WorkspaceGitServiceImpl } from "./workspace-git-service.js"; import { WorkspaceGitServiceImpl } from "./workspace-git-service.js";
import { resolveWorkspaceIdForPath } from "./resolve-workspace-id-for-path.js"; import { resolveWorkspaceIdForPath } from "./resolve-workspace-id-for-path.js";
import { import {
@@ -330,6 +332,7 @@ export interface PaseoDaemonConfig {
trustedProxies?: true | string[]; trustedProxies?: true | string[];
mcpEnabled?: boolean; mcpEnabled?: boolean;
mcpInjectIntoAgents?: boolean; mcpInjectIntoAgents?: boolean;
browserToolsEnabled?: boolean;
autoArchiveAfterMerge?: boolean; autoArchiveAfterMerge?: boolean;
enableTerminalAgentHooks?: boolean; enableTerminalAgentHooks?: boolean;
appendSystemPrompt?: string; appendSystemPrompt?: string;
@@ -383,6 +386,7 @@ export interface PaseoDaemon {
terminalManager: TerminalManager; terminalManager: TerminalManager;
serviceProxy: ServiceProxySubsystem; serviceProxy: ServiceProxySubsystem;
scriptRuntimeStore: WorkspaceScriptRuntimeStore; scriptRuntimeStore: WorkspaceScriptRuntimeStore;
browserToolsBroker: BrowserToolsBroker;
start(): Promise<void>; start(): Promise<void>;
stop(): Promise<void>; stop(): Promise<void>;
getListenTarget(): ListenTarget | null; getListenTarget(): ListenTarget | null;
@@ -429,6 +433,39 @@ function resolveExpressTrustProxySetting(config: PaseoDaemonConfig): true | stri
return config.trustedProxies ?? ["loopback"]; return config.trustedProxies ?? ["loopback"];
} }
function createInitialMutableDaemonConfig(config: PaseoDaemonConfig): MutableDaemonConfig {
const providers: MutableDaemonConfig["providers"] = Object.fromEntries(
Object.entries(config.providerOverrides ?? {}).map(([providerId, override]) => {
const providerConfig: MutableDaemonConfig["providers"][string] = {};
if (override.enabled !== undefined) {
providerConfig.enabled = override.enabled;
}
if (override.additionalModels) {
providerConfig.additionalModels = override.additionalModels;
}
return [providerId, providerConfig];
}),
);
const initialConfig: MutableDaemonConfig = {
mcp: { injectIntoAgents: config.mcpInjectIntoAgents ?? true },
browserTools: { enabled: config.browserToolsEnabled ?? false },
providers,
metadataGeneration: {
providers: config.metadataGeneration?.providers ?? [],
},
autoArchiveAfterMerge: config.autoArchiveAfterMerge ?? false,
enableTerminalAgentHooks: config.enableTerminalAgentHooks ?? false,
appendSystemPrompt: config.appendSystemPrompt ?? "",
};
if (config.terminalProfiles !== undefined) {
initialConfig.terminalProfiles = config.terminalProfiles;
}
return initialConfig;
}
export async function createPaseoDaemon( export async function createPaseoDaemon(
config: PaseoDaemonConfig, config: PaseoDaemonConfig,
rootLogger: Logger, rootLogger: Logger,
@@ -439,29 +476,13 @@ export async function createPaseoDaemon(
const daemonVersion = resolveDaemonVersion(import.meta.url); const daemonVersion = resolveDaemonVersion(import.meta.url);
const daemonConfigStore = new DaemonConfigStore( const daemonConfigStore = new DaemonConfigStore(
config.paseoHome, config.paseoHome,
{ createInitialMutableDaemonConfig(config),
mcp: { injectIntoAgents: config.mcpInjectIntoAgents ?? true },
providers: Object.fromEntries(
Object.entries(config.providerOverrides ?? {}).map(([providerId, override]) => [
providerId,
{
...(override.enabled !== undefined ? { enabled: override.enabled } : {}),
...(override.additionalModels ? { additionalModels: override.additionalModels } : {}),
},
]),
),
metadataGeneration: {
providers: config.metadataGeneration?.providers ?? [],
},
autoArchiveAfterMerge: config.autoArchiveAfterMerge ?? false,
enableTerminalAgentHooks: config.enableTerminalAgentHooks ?? false,
appendSystemPrompt: config.appendSystemPrompt ?? "",
...(config.terminalProfiles !== undefined
? { terminalProfiles: config.terminalProfiles }
: {}),
},
logger, logger,
); );
const browserToolsPolicy = new DaemonConfigBrowserToolsPolicy(daemonConfigStore);
const browserToolsBroker = new BrowserToolsBroker({
policy: browserToolsPolicy,
});
const serverId = getOrCreateServerId(config.paseoHome, { logger }); const serverId = getOrCreateServerId(config.paseoHome, { logger });
const daemonKeyPair = await loadOrCreateDaemonKeyPair(config.paseoHome, logger); const daemonKeyPair = await loadOrCreateDaemonKeyPair(config.paseoHome, logger);
@@ -959,6 +980,7 @@ export async function createPaseoDaemon(
clearWorkspaceArchiving: clearWorkspaceArchivingExternal, clearWorkspaceArchiving: clearWorkspaceArchivingExternal,
ensureWorkspaceForCreate: ensureWorkspaceForCreateExternal, ensureWorkspaceForCreate: ensureWorkspaceForCreateExternal,
createPaseoWorktree: createPaseoWorktreeForTools, createPaseoWorktree: createPaseoWorktreeForTools,
browserToolsBroker,
paseoHome: config.paseoHome, paseoHome: config.paseoHome,
worktreesRoot: config.worktreesRoot, worktreesRoot: config.worktreesRoot,
callerAgentId: runtime.callerAgentId, callerAgentId: runtime.callerAgentId,
@@ -1228,6 +1250,7 @@ export async function createPaseoDaemon(
}, },
}, },
serviceProxyPublicBaseUrl, serviceProxyPublicBaseUrl,
browserToolsBroker,
); );
if (relayEnabled) { if (relayEnabled) {
@@ -1327,6 +1350,7 @@ export async function createPaseoDaemon(
terminalManager, terminalManager,
serviceProxy, serviceProxy,
scriptRuntimeStore, scriptRuntimeStore,
browserToolsBroker,
start, start,
stop, stop,
getListenTarget: () => boundListenTarget, getListenTarget: () => boundListenTarget,
@@ -0,0 +1,330 @@
import { afterEach, describe, expect, test, vi } from "vitest";
import type {
BrowserAutomationCommand,
BrowserAutomationExecuteRequest,
BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { BrowserToolsBroker, type BrowserToolsDesktopClient } from "./broker.js";
import { StaticBrowserToolsPolicy } from "./policy.js";
class FakeDesktopClient implements BrowserToolsDesktopClient {
public readonly receivedRequests: BrowserAutomationExecuteRequest[] = [];
public constructor(public readonly id: string) {}
public sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void {
this.receivedRequests.push(request);
}
public resolveLatestWith(
broker: BrowserToolsBroker,
responsePayload: BrowserAutomationExecuteResponse["payload"],
): boolean {
return broker.receiveResponse({
type: "browser.automation.execute.response",
payload: responsePayload,
});
}
}
class FailingDesktopClient implements BrowserToolsDesktopClient {
public readonly id = "desktop-1";
public sendBrowserAutomationRequest(): void {
throw new Error("websocket send failed");
}
}
function createBroker(options: { enabled: boolean; timeoutMs?: number }): BrowserToolsBroker {
return new BrowserToolsBroker({
policy: new StaticBrowserToolsPolicy(options.enabled),
defaultTimeoutMs: options.timeoutMs ?? 100,
createRequestId: () => "req-1",
});
}
function pageInfoCommand(): BrowserAutomationCommand {
return { command: "page_info", args: { browserId: "browser-1" } };
}
describe("BrowserToolsBroker", () => {
afterEach(() => {
vi.useRealTimers();
});
test("disabled returns browser_disabled", async () => {
const broker = createBroker({ enabled: false });
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.",
retryable: false,
},
});
});
test("no capable desktop returns browser_no_desktop", async () => {
const broker = createBroker({ enabled: true });
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
retryable: true,
},
});
});
test("invalid browser requests return structured failures without contacting desktop", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
await expect(
broker.execute({
command: {
command: "new_tab",
args: { url: "ftp://example.com" },
} as unknown as BrowserAutomationCommand,
}),
).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_unknown_error",
message: "Browser automation request is invalid: URL must use http or https.",
retryable: false,
},
});
expect(client.receivedRequests).toEqual([]);
expect(broker.getPendingRequestCount()).toBe(0);
});
test("capable fake desktop receives request and returns response", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
workspaceId: "workspace-1",
});
expect(client.receivedRequests).toEqual([
{
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
},
]);
expect(broker.getPendingRequestCount()).toBe(1);
expect(
client.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
},
],
},
}),
).toBe(true);
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
isActive: false,
isLoading: false,
},
],
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("desktop receives snapshot requests", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
workspaceId: "workspace-1",
});
expect(client.receivedRequests).toEqual([
{
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
},
]);
client.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
elements: [],
},
});
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
elements: [],
},
});
});
test("timeout resolves browser_timeout and clears pending state", async () => {
vi.useFakeTimers();
const broker = createBroker({ enabled: true, timeoutMs: 50 });
broker.registerClient(new FakeDesktopClient("desktop-1"));
const resultPromise = broker.execute({ command: pageInfoCommand() });
expect(broker.getPendingRequestCount()).toBe(1);
await vi.advanceTimersByTimeAsync(50);
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_timeout",
message: "Browser automation timed out after 50ms.",
retryable: true,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("disconnect resolves retryable failure and clears pending request", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
const unregister = broker.registerClient(client);
const resultPromise = broker.execute({ command: pageInfoCommand() });
expect(broker.getPendingRequestCount()).toBe(1);
unregister();
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_no_desktop",
message: "The desktop browser automation client disconnected before responding.",
retryable: true,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("desktop send failure resolves structured failure and clears pending request", async () => {
const broker = createBroker({ enabled: true });
broker.registerClient(new FailingDesktopClient());
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_unknown_error",
message: "Browser automation request failed to send: websocket send failed",
retryable: false,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("explicit browser failure response propagates typed error", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({ command: pageInfoCommand() });
client.resolveLatestWith(broker, {
requestId: "req-1",
ok: false,
error: {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
},
});
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("invalid browser response resolves a structured failure and clears pending state", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({ command: pageInfoCommand() });
expect(broker.getPendingRequestCount()).toBe(1);
expect(
broker.receiveResponse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: true,
result: { command: "future_command" },
},
} as unknown as BrowserAutomationExecuteResponse),
).toBe(true);
await expect(resultPromise).resolves.toMatchObject({
requestId: "req-1",
ok: false,
error: {
code: "browser_unknown_error",
retryable: false,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
});
@@ -0,0 +1,284 @@
import { randomUUID } from "node:crypto";
import {
BrowserAutomationExecuteRequestSchema,
BrowserAutomationExecuteResponseSchema,
type BrowserAutomationCommand,
type BrowserAutomationExecuteRequest,
type BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { browserToolsFailure, type BrowserToolsResponsePayload } from "./errors.js";
import type { BrowserToolsPolicy } from "./policy.js";
export interface BrowserToolsDesktopClient {
id: string;
sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void | Promise<void>;
}
export interface BrowserToolsExecuteInput {
command: BrowserAutomationCommand;
agentId?: string;
cwd?: string;
workspaceId?: string;
browserId?: string;
requestId?: string;
timeoutMs?: number;
}
interface PendingBrowserToolsRequest {
clientId: string;
timeout: ReturnType<typeof setTimeout>;
resolve: (payload: BrowserToolsResponsePayload) => void;
}
export interface BrowserToolsBrokerOptions {
policy: BrowserToolsPolicy;
defaultTimeoutMs?: number;
createRequestId?: () => string;
}
const DEFAULT_BROWSER_TOOLS_TIMEOUT_MS = 15_000;
export class BrowserToolsBroker {
private readonly policy: BrowserToolsPolicy;
private readonly defaultTimeoutMs: number;
private readonly createRequestId: () => string;
private readonly clients = new Map<string, BrowserToolsDesktopClient>();
private readonly pending = new Map<string, PendingBrowserToolsRequest>();
public constructor(options: BrowserToolsBrokerOptions) {
this.policy = options.policy;
this.defaultTimeoutMs = options.defaultTimeoutMs ?? DEFAULT_BROWSER_TOOLS_TIMEOUT_MS;
this.createRequestId = options.createRequestId ?? (() => `browser_${randomUUID()}`);
}
public registerClient(client: BrowserToolsDesktopClient): () => void {
this.clients.set(client.id, client);
return () => this.unregisterClient(client.id);
}
public unregisterClient(clientId: string): void {
const deleted = this.clients.delete(clientId);
if (!deleted) {
return;
}
for (const [requestId, pending] of this.pending) {
if (pending.clientId !== clientId) {
continue;
}
this.pending.delete(requestId);
clearTimeout(pending.timeout);
pending.resolve(
browserToolsFailure({
requestId,
code: "browser_no_desktop",
message: "The desktop browser automation client disconnected before responding.",
retryable: true,
}),
);
}
}
public getPendingRequestCount(): number {
return this.pending.size;
}
public getRegisteredClientCount(): number {
return this.clients.size;
}
public async execute(input: BrowserToolsExecuteInput): Promise<BrowserToolsResponsePayload> {
const requestId = input.requestId ?? this.createRequestId();
if (!this.policy.isEnabled()) {
return browserToolsFailure({
requestId,
code: "browser_disabled",
message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.",
});
}
const client = this.selectClient();
if (!client) {
return browserToolsFailure({
requestId,
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
retryable: true,
});
}
const request = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId,
...(input.agentId ? { agentId: input.agentId } : {}),
...(input.cwd ? { cwd: input.cwd } : {}),
...(input.workspaceId ? { workspaceId: input.workspaceId } : {}),
...(input.browserId ? { browserId: input.browserId } : {}),
command: input.command,
});
if (!request.success) {
return browserToolsFailure({
requestId,
code: "browser_unknown_error",
message: formatBrowserAutomationValidationError(request.error.issues[0]?.message),
});
}
return this.sendRequest({
client,
request: request.data,
timeoutMs: input.timeoutMs ?? this.defaultTimeoutMs,
});
}
public receiveResponse(response: BrowserAutomationExecuteResponse): boolean {
const parsed = BrowserAutomationExecuteResponseSchema.safeParse(response);
if (!parsed.success) {
const requestId = getBrowserAutomationResponseRequestId(response);
if (!requestId) {
return false;
}
const pending = this.pending.get(requestId);
if (!pending) {
return false;
}
this.pending.delete(requestId);
clearTimeout(pending.timeout);
pending.resolve(
browserToolsFailure({
requestId,
code: "browser_unknown_error",
message: formatBrowserAutomationResponseValidationError(parsed.error.issues[0]?.message),
}),
);
return true;
}
const pending = this.pending.get(parsed.data.payload.requestId);
if (!pending) {
return false;
}
this.pending.delete(parsed.data.payload.requestId);
clearTimeout(pending.timeout);
pending.resolve(parsed.data.payload);
return true;
}
private selectClient(): BrowserToolsDesktopClient | null {
for (const client of this.clients.values()) {
return client;
}
return null;
}
private sendRequest(params: {
client: BrowserToolsDesktopClient;
request: BrowserAutomationExecuteRequest;
timeoutMs: number;
}): Promise<BrowserToolsResponsePayload> {
const { client, request, timeoutMs } = params;
return new Promise<BrowserToolsResponsePayload>((resolve) => {
const timeout = setTimeout(() => {
if (!this.pending.delete(request.requestId)) {
return;
}
resolve(
browserToolsFailure({
requestId: request.requestId,
code: "browser_timeout",
message: `Browser automation timed out after ${timeoutMs}ms.`,
retryable: true,
}),
);
}, timeoutMs);
this.pending.set(request.requestId, {
clientId: client.id,
timeout,
resolve,
});
try {
Promise.resolve(client.sendBrowserAutomationRequest(request)).catch((error: unknown) => {
resolveSendFailure({
requestId: request.requestId,
pending: this.pending,
timeout,
resolve,
error,
});
});
} catch (error) {
resolveSendFailure({
requestId: request.requestId,
pending: this.pending,
timeout,
resolve,
error,
});
}
});
}
}
function resolveSendFailure(params: {
requestId: string;
pending: Map<string, PendingBrowserToolsRequest>;
timeout: ReturnType<typeof setTimeout>;
resolve: (payload: BrowserToolsResponsePayload) => void;
error: unknown;
}): void {
if (!params.pending.delete(params.requestId)) {
return;
}
clearTimeout(params.timeout);
params.resolve(
browserToolsFailure({
requestId: params.requestId,
code: "browser_unknown_error",
message: formatBrowserAutomationSendError(params.error),
}),
);
}
function formatBrowserAutomationValidationError(message: string | undefined): string {
if (!message) {
return "Browser automation request is invalid.";
}
return `Browser automation request is invalid: ${message}.`;
}
function formatBrowserAutomationResponseValidationError(message: string | undefined): string {
if (!message) {
return "Browser automation response is invalid.";
}
return `Browser automation response is invalid: ${message}.`;
}
function formatBrowserAutomationSendError(error: unknown): string {
if (error instanceof Error && error.message) {
return `Browser automation request failed to send: ${error.message}`;
}
return `Browser automation request failed to send: ${String(error)}`;
}
function getBrowserAutomationResponseRequestId(response: unknown): string | null {
if (!isRecord(response)) {
return null;
}
const payload = response.payload;
if (!isRecord(payload) || typeof payload.requestId !== "string") {
return null;
}
return payload.requestId;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
@@ -0,0 +1,48 @@
import type {
BrowserAutomationErrorCode,
BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
export type BrowserToolsResponsePayload = BrowserAutomationExecuteResponse["payload"];
export type BrowserToolsErrorPayload = Extract<BrowserToolsResponsePayload, { ok: false }>["error"];
export class BrowserToolsRequestError extends Error {
public readonly code: BrowserAutomationErrorCode;
public readonly retryable: boolean;
public constructor(error: BrowserToolsErrorPayload) {
super(error.message);
this.name = "BrowserToolsRequestError";
this.code = error.code;
this.retryable = error.retryable;
}
}
export function browserToolsFailure(params: {
requestId: string;
code: BrowserAutomationErrorCode;
message: string;
retryable?: boolean;
}): BrowserToolsResponsePayload {
return {
requestId: params.requestId,
ok: false,
error: {
code: params.code,
message: params.message,
retryable: params.retryable ?? false,
},
};
}
export function createBrowserToolsRequestError(params: {
code: BrowserAutomationErrorCode;
message: string;
retryable?: boolean;
}): BrowserToolsRequestError {
return new BrowserToolsRequestError({
code: params.code,
message: params.message,
retryable: params.retryable ?? false,
});
}
@@ -0,0 +1,29 @@
import type { DaemonConfigStore, MutableDaemonConfig } from "../daemon-config-store.js";
export interface BrowserToolsPolicy {
isEnabled(): boolean;
}
export class StaticBrowserToolsPolicy implements BrowserToolsPolicy {
public constructor(private readonly enabled: boolean) {}
public isEnabled(): boolean {
return this.enabled;
}
}
export class DaemonConfigBrowserToolsPolicy implements BrowserToolsPolicy {
public constructor(private readonly configStore: Pick<DaemonConfigStore, "get">) {}
public isEnabled(): boolean {
return readBrowserToolsEnabled(this.configStore.get());
}
}
function readBrowserToolsEnabled(config: MutableDaemonConfig): boolean {
const browserTools = config.browserTools;
if (typeof browserTools !== "object" || browserTools === null || Array.isArray(browserTools)) {
return false;
}
return browserTools.enabled === true;
}
@@ -0,0 +1,850 @@
import { describe, expect, it, vi } from "vitest";
import type { BrowserToolsBroker } from "./broker.js";
import type { BrowserToolsResponsePayload } from "./errors.js";
import { registerBrowserTools, type RegisterBrowserToolsOptions } from "./tools.js";
interface RegisteredTool {
config: { inputSchema: Record<string, unknown>; outputSchema: unknown };
handler: (args: Record<string, unknown>) => Promise<{
content: Array<{ type: string; text?: string }>;
structuredContent?: Record<string, unknown>;
}>;
}
function createHarness(options?: { brokerResponse?: BrowserToolsResponsePayload }) {
const tools = new Map<string, RegisteredTool>();
const execute = vi.fn(async () => {
return (
options?.brokerResponse ?? {
requestId: "req-1",
ok: true as const,
result: {
command: "list_tabs" as const,
tabs: [
{
browserId: "browser-1",
url: "https://example.com",
title: "Example",
isActive: true,
isLoading: false,
},
],
},
}
);
});
const registerTool = vi.fn((name, config, handler) => {
tools.set(name, {
config,
handler: handler as RegisteredTool["handler"],
});
}) as unknown as RegisterBrowserToolsOptions["registerTool"];
registerBrowserTools({
registerTool,
broker: { execute } as unknown as BrowserToolsBroker,
callerAgentId: "agent-1",
resolveCallerAgent: () => ({ id: "agent-1", cwd: "/repo" }),
});
return { tools, execute };
}
function tool(harness: ReturnType<typeof createHarness>, name: string): RegisteredTool {
const registered = harness.tools.get(name);
if (!registered) {
throw new Error(`Tool not registered: ${name}`);
}
return registered;
}
describe("registerBrowserTools", () => {
it("registers browser_list_tabs and routes through the broker with caller workspace", async () => {
const harness = createHarness();
const response = await tool(harness, "browser_list_tabs").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "list_tabs", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([
{
type: "text",
text: "Found 1 Paseo browser tab. Use these browserId values exactly; do not use 'default'. You may omit browserId to use the active tab.\n- browserId=browser-1 active title=\"Example\" url=https://example.com",
},
]);
expect(response.structuredContent).toEqual({
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: "browser-1",
url: "https://example.com",
title: "Example",
isActive: true,
isLoading: false,
},
],
},
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("registers browser_new_tab and returns the created tab handle", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-new-tab",
ok: true,
result: {
command: "new_tab",
browserId: "browser-new",
workspaceId: "/repo",
url: "https://example.com",
},
},
});
const response = await tool(harness, "browser_new_tab").handler({ url: "https://example.com" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "new_tab",
args: { workspaceId: "/repo", url: "https://example.com" },
},
});
expect(response.content).toEqual([
{
type: "text",
text: "Created browser tab browserId=browser-new url=https://example.com. Use this browserId exactly, or omit browserId to use this active tab.",
},
]);
});
it("routes browser_page_info through the broker with explicit browserId", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-2",
ok: true,
result: {
command: "page_info",
tab: {
browserId: "browser-2",
url: "https://example.com/docs",
title: "Docs",
isActive: false,
isLoading: false,
},
},
},
});
const response = await tool(harness, "browser_page_info").handler({ browserId: "browser-2" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
browserId: "browser-2",
command: {
command: "page_info",
args: { workspaceId: "/repo", browserId: "browser-2" },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Current page browserId=browser-2: Docs — https://example.com/docs" },
]);
expect(response.structuredContent).toEqual({
ok: true,
result: {
command: "page_info",
tab: {
browserId: "browser-2",
url: "https://example.com/docs",
title: "Docs",
isActive: false,
isLoading: false,
},
},
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo", browserId: "browser-2" },
});
});
it("routes browser_snapshot through the broker with caller workspace", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-snapshot",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "/repo",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name" },
},
],
},
},
});
const response = await tool(harness, "browser_snapshot").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "snapshot", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([{ type: "text", text: "Snapshot captured 1 element." }]);
expect(response.structuredContent).toEqual({
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "/repo",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name" },
},
],
},
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("routes browser_set_background through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-bg",
ok: true,
result: { command: "set_background", browserId: "browser-1", color: "red" },
},
});
const response = await tool(harness, "browser_set_background").handler({ color: "red" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "set_background", args: { workspaceId: "/repo", color: "red" } },
});
expect(response.content).toEqual([
{ type: "text", text: "Set browser page background to red." },
]);
});
it("routes browser_click through the broker with a snapshot ref", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-click",
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e2" },
},
});
const response = await tool(harness, "browser_click").handler({ ref: "@e2" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "click", args: { workspaceId: "/repo", ref: "@e2" } },
});
expect(response.content).toEqual([{ type: "text", text: "Clicked browser element @e2." }]);
expect(response.structuredContent).toEqual({
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e2" },
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("routes browser_fill through the broker with a snapshot ref and value", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-fill",
ok: true,
result: { command: "fill", browserId: "browser-1", ref: "@e1" },
},
});
const response = await tool(harness, "browser_fill").handler({ ref: "@e1", value: "Ada" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "fill", args: { workspaceId: "/repo", ref: "@e1", value: "Ada" } },
});
expect(response.content).toEqual([{ type: "text", text: "Filled browser element @e1." }]);
expect(response.structuredContent).toEqual({
ok: true,
result: { command: "fill", browserId: "browser-1", ref: "@e1" },
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("routes browser_wait through the broker with text and timeout", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-wait",
ok: true,
result: { command: "wait", browserId: "browser-1", matched: "text" },
},
});
const response = await tool(harness, "browser_wait").handler({
text: "Ready",
timeoutMs: 1000,
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
timeoutMs: 2000,
command: {
command: "wait",
args: { workspaceId: "/repo", text: "Ready", timeoutMs: 1000 },
},
});
expect(response.content).toEqual([{ type: "text", text: "Browser wait matched text." }]);
expect(response.structuredContent).toEqual({
ok: true,
result: { command: "wait", browserId: "browser-1", matched: "text" },
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("routes browser_type through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-type",
ok: true,
result: { command: "type", browserId: "browser-1", ref: "@e1" },
},
});
const response = await tool(harness, "browser_type").handler({ ref: "@e1", text: "Ada" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "type", args: { workspaceId: "/repo", ref: "@e1", text: "Ada" } },
});
expect(response.content).toEqual([{ type: "text", text: "Typed into browser element @e1." }]);
});
it("routes browser_keypress through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-keypress",
ok: true,
result: { command: "keypress", browserId: "browser-1", ref: "@e1", key: "Enter" },
},
});
const response = await tool(harness, "browser_keypress").handler({
ref: "@e1",
key: "Enter",
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "keypress",
args: { workspaceId: "/repo", ref: "@e1", key: "Enter" },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Pressed Enter on browser element @e1." },
]);
});
it("routes browser_navigate through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-nav",
ok: true,
result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" },
},
});
const response = await tool(harness, "browser_navigate").handler({
url: "https://example.com/next",
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "navigate",
args: { workspaceId: "/repo", url: "https://example.com/next" },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Navigated browser to https://example.com/next." },
]);
});
it("routes browser_back through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-back",
ok: true,
result: { command: "back", browserId: "browser-1" },
},
});
const response = await tool(harness, "browser_back").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "back", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([{ type: "text", text: "Browser back complete." }]);
});
it("routes browser_screenshot through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-shot",
ok: true,
result: {
command: "screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 100,
height: 50,
},
},
});
const response = await tool(harness, "browser_screenshot").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "screenshot", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([
{ type: "text", text: "Captured browser screenshot (100x50)." },
{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
]);
});
it("routes browser_logs through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-logs",
ok: true,
result: {
command: "logs",
browserId: "browser-1",
console: [{ level: "info", message: "ready", timestamp: 1 }],
network: [
{
url: "https://example.com/app.js",
type: "script",
startTime: 2,
duration: 3,
},
],
},
},
});
const response = await tool(harness, "browser_logs").handler({ maxEntries: 25 });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "logs", args: { workspaceId: "/repo", maxEntries: 25 } },
});
expect(response.content).toEqual([
{ type: "text", text: "Read 1 console log and 1 network entry." },
]);
});
it("routes browser_storage through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-storage",
ok: true,
result: {
command: "storage",
browserId: "browser-1",
url: "https://example.com",
cookies: [{ name: "theme", value: "dark" }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
},
});
const response = await tool(harness, "browser_storage").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "storage", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([
{ type: "text", text: "Read 1 cookie, 1 localStorage entry, and 1 sessionStorage entry." },
]);
});
it("routes browser_environment through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-environment",
ok: true,
result: {
command: "environment",
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
});
const response = await tool(harness, "browser_environment").handler({
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "environment",
args: {
workspaceId: "/repo",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
});
expect(response.content).toEqual([
{ type: "text", text: "Browser environment viewport is 390x844." },
]);
});
it("routes browser_full_page_screenshot through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-full-page",
ok: true,
result: {
command: "full_page_screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 390,
height: 1200,
},
},
});
const response = await tool(harness, "browser_full_page_screenshot").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "full_page_screenshot", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([
{ type: "text", text: "Captured full-page browser screenshot (390x1200)." },
{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
]);
});
it("routes browser_pdf through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-pdf",
ok: true,
result: {
command: "pdf",
browserId: "browser-1",
mimeType: "application/pdf",
dataBase64: "JVBERg==",
},
},
});
const response = await tool(harness, "browser_pdf").handler({ landscape: true });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "pdf",
args: { workspaceId: "/repo", landscape: true, printBackground: true },
},
});
expect(response.content).toEqual([{ type: "text", text: "Exported browser page PDF." }]);
});
it("routes browser_download through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-download",
ok: true,
result: {
command: "download",
browserId: "browser-1",
url: "https://example.com/file.txt",
filePath: "/tmp/file.txt",
totalBytes: 5,
state: "completed",
},
},
});
const response = await tool(harness, "browser_download").handler({
url: "https://example.com/file.txt",
fileName: "file.txt",
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "download",
args: { workspaceId: "/repo", url: "https://example.com/file.txt", fileName: "file.txt" },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Downloaded browser file to /tmp/file.txt." },
]);
});
it("routes browser_upload through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-upload",
ok: true,
result: {
command: "upload",
browserId: "browser-1",
ref: "@e1",
filePaths: ["/tmp/file.txt"],
},
},
});
const response = await tool(harness, "browser_upload").handler({
ref: "@e1",
filePaths: ["/tmp/file.txt"],
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "upload",
args: { workspaceId: "/repo", ref: "@e1", filePaths: ["/tmp/file.txt"] },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Uploaded 1 file to browser element @e1." },
]);
});
it("routes browser form control tools through the broker", async () => {
const focusHarness = createHarness({
brokerResponse: {
requestId: "req-focus",
ok: true,
result: { command: "focus", browserId: "browser-1", ref: "@e1" },
},
});
const focusResponse = await tool(focusHarness, "browser_focus").handler({ ref: "@e1" });
expect(focusHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "focus", args: { workspaceId: "/repo", ref: "@e1" } },
});
expect(focusResponse.content).toEqual([{ type: "text", text: "Focused browser element @e1." }]);
const checkHarness = createHarness({
brokerResponse: {
requestId: "req-check",
ok: true,
result: { command: "check", browserId: "browser-1", ref: "@e2", checked: false },
},
});
const checkResponse = await tool(checkHarness, "browser_check").handler({
ref: "@e2",
checked: false,
});
expect(checkHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "check", args: { workspaceId: "/repo", ref: "@e2", checked: false } },
});
expect(checkResponse.content).toEqual([
{ type: "text", text: "Unchecked browser element @e2." },
]);
const selectHarness = createHarness({
brokerResponse: {
requestId: "req-select",
ok: true,
result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" },
},
});
const selectResponse = await tool(selectHarness, "browser_select").handler({
ref: "@e3",
value: "us",
});
expect(selectHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "select", args: { workspaceId: "/repo", ref: "@e3", value: "us" } },
});
expect(selectResponse.content).toEqual([
{ type: "text", text: "Selected us in browser element @e3." },
]);
const hoverHarness = createHarness({
brokerResponse: {
requestId: "req-hover",
ok: true,
result: { command: "hover", browserId: "browser-1", ref: "@e4" },
},
});
const hoverResponse = await tool(hoverHarness, "browser_hover").handler({ ref: "@e4" });
expect(hoverHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "hover", args: { workspaceId: "/repo", ref: "@e4" } },
});
expect(hoverResponse.content).toEqual([{ type: "text", text: "Hovered browser element @e4." }]);
const dragHarness = createHarness({
brokerResponse: {
requestId: "req-drag",
ok: true,
result: { command: "drag", browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" },
},
});
const dragResponse = await tool(dragHarness, "browser_drag").handler({
sourceRef: "@e4",
targetRef: "@e5",
});
expect(dragHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "drag",
args: { workspaceId: "/repo", sourceRef: "@e4", targetRef: "@e5" },
},
});
expect(dragResponse.content).toEqual([
{ type: "text", text: "Dragged browser element @e4 to @e5." },
]);
});
it("returns model-actionable disabled errors with structured content", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-disabled",
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.",
retryable: false,
},
},
});
const response = await tool(harness, "browser_list_tabs").handler({});
expect(response.content).toEqual([
{
type: "text",
text: "Browser tools are disabled. Enable desktop browser tools on the host, then try again.",
},
]);
expect(response.structuredContent).toEqual({
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.",
retryable: false,
},
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("preserves typed broker errors", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-timeout",
ok: false,
error: {
code: "browser_timeout",
message: "Browser automation timed out after 15000ms.",
retryable: true,
},
},
});
const response = await tool(harness, "browser_page_info").handler({});
expect(response.content).toEqual([
{
type: "text",
text: "The browser did not respond before the timeout. Try again or check the desktop app.",
},
]);
expect(response.structuredContent?.error).toEqual({
code: "browser_timeout",
message: "Browser automation timed out after 15000ms.",
retryable: true,
});
});
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,38 @@
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, test } from "vitest";
import { loadConfig } from "./config.js";
const roots: string[] = [];
async function createPaseoHome(config: unknown): Promise<string> {
const root = await mkdtemp(path.join(os.tmpdir(), "paseo-config-browser-tools-"));
roots.push(root);
const paseoHome = path.join(root, ".paseo");
await mkdir(paseoHome, { recursive: true });
await writeFile(path.join(paseoHome, "config.json"), JSON.stringify(config, null, 2));
return paseoHome;
}
describe("daemon browser tools config", () => {
afterEach(async () => {
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
});
test("defaults browser tools off when config is absent", async () => {
const home = await createPaseoHome({ version: 1 });
expect(loadConfig(home, { env: {} }).browserToolsEnabled).toBe(false);
});
test("loads browser tools opt-in from persisted daemon config", async () => {
const home = await createPaseoHome({
version: 1,
daemon: { browserTools: { enabled: true } },
});
expect(loadConfig(home, { env: {} }).browserToolsEnabled).toBe(true);
});
});
+7
View File
@@ -399,6 +399,10 @@ function resolveAppendSystemPrompt(persisted: ReturnType<typeof loadPersistedCon
return persisted.daemon?.appendSystemPrompt ?? ""; return persisted.daemon?.appendSystemPrompt ?? "";
} }
function resolveBrowserToolsEnabled(persisted: ReturnType<typeof loadPersistedConfig>): boolean {
return persisted.daemon?.browserTools?.enabled ?? false;
}
function resolveStaticLoadConfigSettings( function resolveStaticLoadConfigSettings(
env: NodeJS.ProcessEnv, env: NodeJS.ProcessEnv,
cli: CliConfigOverrides | undefined, cli: CliConfigOverrides | undefined,
@@ -408,6 +412,7 @@ function resolveStaticLoadConfigSettings(
mcpEnabled: cli?.mcpEnabled ?? persisted.daemon?.mcp?.enabled ?? true, mcpEnabled: cli?.mcpEnabled ?? persisted.daemon?.mcp?.enabled ?? true,
mcpInjectIntoAgents: mcpInjectIntoAgents:
cli?.mcpInjectIntoAgents ?? persisted.daemon?.mcp?.injectIntoAgents ?? false, cli?.mcpInjectIntoAgents ?? persisted.daemon?.mcp?.injectIntoAgents ?? false,
browserToolsEnabled: resolveBrowserToolsEnabled(persisted),
autoArchiveAfterMerge: persisted.daemon?.autoArchiveAfterMerge ?? false, autoArchiveAfterMerge: persisted.daemon?.autoArchiveAfterMerge ?? false,
appendSystemPrompt: resolveAppendSystemPrompt(persisted), appendSystemPrompt: resolveAppendSystemPrompt(persisted),
terminalProfiles: persisted.daemon?.terminalProfiles, terminalProfiles: persisted.daemon?.terminalProfiles,
@@ -435,6 +440,7 @@ export function loadConfig(
const { const {
mcpEnabled, mcpEnabled,
mcpInjectIntoAgents, mcpInjectIntoAgents,
browserToolsEnabled,
autoArchiveAfterMerge, autoArchiveAfterMerge,
appendSystemPrompt, appendSystemPrompt,
terminalProfiles, terminalProfiles,
@@ -472,6 +478,7 @@ export function loadConfig(
trustedProxies, trustedProxies,
mcpEnabled, mcpEnabled,
mcpInjectIntoAgents, mcpInjectIntoAgents,
browserToolsEnabled,
autoArchiveAfterMerge, autoArchiveAfterMerge,
enableTerminalAgentHooks: persisted.daemon?.enableTerminalAgentHooks ?? false, enableTerminalAgentHooks: persisted.daemon?.enableTerminalAgentHooks ?? false,
appendSystemPrompt, appendSystemPrompt,
@@ -94,6 +94,7 @@ describe("DaemonConfigStore", () => {
paseoHome, paseoHome,
{ {
mcp: { injectIntoAgents: false }, mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {}, providers: {},
metadataGeneration: { providers: [] }, metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
@@ -126,6 +127,7 @@ describe("DaemonConfigStore", () => {
paseoHome, paseoHome,
{ {
mcp: { injectIntoAgents: false }, mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {}, providers: {},
metadataGeneration: { providers: [] }, metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
@@ -143,6 +145,29 @@ describe("DaemonConfigStore", () => {
expect(persisted.daemon?.appendSystemPrompt).toBe("Prefer terse replies."); expect(persisted.daemon?.appendSystemPrompt).toBe("Prefer terse replies.");
}); });
test("patch persists browser tools opt-in into config.json", () => {
const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-store-"));
tempDirs.push(paseoHome);
const store = new DaemonConfigStore(
paseoHome,
{
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {},
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
appendSystemPrompt: "",
},
undefined,
);
store.patch({ browserTools: { enabled: true } });
const persisted = loadPersistedConfig(paseoHome);
expect(persisted.daemon?.browserTools).toEqual({ enabled: true });
});
test("patch persists provider additional models into config.json", () => { test("patch persists provider additional models into config.json", () => {
const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-store-")); const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-store-"));
tempDirs.push(paseoHome); tempDirs.push(paseoHome);
@@ -151,6 +176,7 @@ describe("DaemonConfigStore", () => {
paseoHome, paseoHome,
{ {
mcp: { injectIntoAgents: false }, mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {}, providers: {},
metadataGeneration: { providers: [] }, metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
@@ -192,6 +218,7 @@ describe("DaemonConfigStore", () => {
paseoHome, paseoHome,
{ {
mcp: { injectIntoAgents: false }, mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {}, providers: {},
metadataGeneration: { providers: [] }, metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
@@ -240,6 +267,7 @@ describe("DaemonConfigStore", () => {
paseoHome, paseoHome,
{ {
mcp: { injectIntoAgents: false }, mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {}, providers: {},
metadataGeneration: { providers: [] }, metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
@@ -292,6 +320,7 @@ describe("DaemonConfigStore", () => {
paseoHome, paseoHome,
{ {
mcp: { injectIntoAgents: false }, mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {}, providers: {},
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
enableTerminalAgentHooks: false, enableTerminalAgentHooks: false,
@@ -315,6 +344,7 @@ describe("DaemonConfigStore", () => {
paseoHome, paseoHome,
{ {
mcp: { injectIntoAgents: false }, mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {}, providers: {},
autoArchiveAfterMerge: false, autoArchiveAfterMerge: false,
enableTerminalAgentHooks: false, enableTerminalAgentHooks: false,
@@ -172,6 +172,7 @@ function mergeMutableConfigIntoPersistedConfig(params: {
mutable: MutableDaemonConfig; mutable: MutableDaemonConfig;
}): PersistedConfig { }): PersistedConfig {
const { persisted, mutable } = params; const { persisted, mutable } = params;
const browserToolsEnabled = readBrowserToolsEnabled(mutable);
const metadataGenerationProviders = readMetadataGenerationProviders(mutable); const metadataGenerationProviders = readMetadataGenerationProviders(mutable);
const providerOverrides = applyMutableProviderConfigToOverrides( const providerOverrides = applyMutableProviderConfigToOverrides(
persisted.agents?.providers as Record<string, ProviderOverride> | undefined, persisted.agents?.providers as Record<string, ProviderOverride> | undefined,
@@ -208,6 +209,10 @@ function mergeMutableConfigIntoPersistedConfig(params: {
...persisted.daemon?.mcp, ...persisted.daemon?.mcp,
injectIntoAgents: mutable.mcp.injectIntoAgents, injectIntoAgents: mutable.mcp.injectIntoAgents,
}, },
browserTools: {
...persisted.daemon?.browserTools,
enabled: browserToolsEnabled,
},
autoArchiveAfterMerge: mutable.autoArchiveAfterMerge, autoArchiveAfterMerge: mutable.autoArchiveAfterMerge,
enableTerminalAgentHooks: mutable.enableTerminalAgentHooks, enableTerminalAgentHooks: mutable.enableTerminalAgentHooks,
appendSystemPrompt: mutable.appendSystemPrompt, appendSystemPrompt: mutable.appendSystemPrompt,
@@ -219,6 +224,14 @@ function mergeMutableConfigIntoPersistedConfig(params: {
} as PersistedConfig; } as PersistedConfig;
} }
function readBrowserToolsEnabled(mutable: MutableDaemonConfig): boolean {
const browserTools = mutable.browserTools;
if (!isRecord(browserTools)) {
return false;
}
return browserTools["enabled"] === true;
}
function readMetadataGenerationProviders( function readMetadataGenerationProviders(
mutable: MutableDaemonConfig, mutable: MutableDaemonConfig,
): Array<{ provider: string; model?: string; thinkingOptionId?: string }> { ): Array<{ provider: string; model?: string; thinkingOptionId?: string }> {
@@ -46,6 +46,18 @@ describe("PersistedConfigSchema daemon append system prompt config", () => {
}); });
}); });
describe("PersistedConfigSchema daemon browser tools config", () => {
test("accepts optional browser tools opt-in", () => {
const parsed = PersistedConfigSchema.parse({
daemon: {
browserTools: { enabled: true },
},
});
expect(parsed.daemon?.browserTools?.enabled).toBe(true);
});
});
describe("PersistedConfigSchema daemon relay config", () => { describe("PersistedConfigSchema daemon relay config", () => {
test("accepts optional relay TLS setting", () => { test("accepts optional relay TLS setting", () => {
const parsed = PersistedConfigSchema.parse({ const parsed = PersistedConfigSchema.parse({
@@ -240,6 +240,12 @@ export const PersistedConfigSchema = z
}) })
.passthrough() .passthrough()
.optional(), .optional(),
browserTools: z
.object({
enabled: z.boolean().optional(),
})
.passthrough()
.optional(),
autoArchiveAfterMerge: z.boolean().optional(), autoArchiveAfterMerge: z.boolean().optional(),
enableTerminalAgentHooks: z.boolean().optional(), enableTerminalAgentHooks: z.boolean().optional(),
appendSystemPrompt: z.string().optional(), appendSystemPrompt: z.string().optional(),
@@ -3816,7 +3816,7 @@ test("open_project_request does not match a new child directory to an existing p
const session = createSessionForWorkspaceTests(); const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>(); const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>(); const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const home = path.resolve("/Users/moboudra"); const home = path.resolve("/home/developer");
const worktree = path.join(home, ".paseo", "worktrees", "project-config-lifecycle-textarea"); const worktree = path.join(home, ".paseo", "worktrees", "project-config-lifecycle-textarea");
projects.set( projects.set(
@@ -3825,7 +3825,7 @@ test("open_project_request does not match a new child directory to an existing p
projectId: home, projectId: home,
rootPath: home, rootPath: home,
kind: "non_git", kind: "non_git",
displayName: "moboudra", displayName: "developer",
createdAt: "2026-04-24T09:00:00.000Z", createdAt: "2026-04-24T09:00:00.000Z",
updatedAt: "2026-04-24T09:00:00.000Z", updatedAt: "2026-04-24T09:00:00.000Z",
}), }),
@@ -3837,7 +3837,7 @@ test("open_project_request does not match a new child directory to an existing p
projectId: home, projectId: home,
cwd: home, cwd: home,
kind: "directory", kind: "directory",
displayName: "moboudra", displayName: "developer",
createdAt: "2026-04-24T09:00:00.000Z", createdAt: "2026-04-24T09:00:00.000Z",
updatedAt: "2026-04-24T09:00:00.000Z", updatedAt: "2026-04-24T09:00:00.000Z",
}), }),
@@ -3882,7 +3882,7 @@ test("open_project_request does not unarchive an archived parent workspace for a
const session = createSessionForWorkspaceTests(); const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>(); const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>(); const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const home = path.resolve("/Users/moboudra"); const home = path.resolve("/home/developer");
const worktree = path.join(home, ".paseo", "worktrees", "project-config-lifecycle-textarea"); const worktree = path.join(home, ".paseo", "worktrees", "project-config-lifecycle-textarea");
const archivedAt = "2026-04-24T08:00:00.000Z"; const archivedAt = "2026-04-24T08:00:00.000Z";
@@ -3950,9 +3950,9 @@ test("open_project_request reclassifies an archived directory workspace when git
const session = createSessionForWorkspaceTests(); const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>(); const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>(); const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const repoRoot = path.resolve("/Users/moboudra/dev/paseo"); const repoRoot = path.resolve("/home/developer/dev/paseo");
const cwd = path.join( const cwd = path.join(
path.resolve("/Users/moboudra"), path.resolve("/home/developer"),
".paseo", ".paseo",
"worktrees", "worktrees",
"orchestrate", "orchestrate",
@@ -4049,9 +4049,9 @@ test("open_project_request reclassifies an active directory workspace when git m
const session = createSessionForWorkspaceTests(); const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>(); const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>(); const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const repoRoot = path.resolve("/Users/moboudra/dev/paseo"); const repoRoot = path.resolve("/home/developer/dev/paseo");
const cwd = path.join( const cwd = path.join(
path.resolve("/Users/moboudra"), path.resolve("/home/developer"),
".paseo", ".paseo",
"worktrees", "worktrees",
"orchestrate", "orchestrate",
@@ -4167,9 +4167,9 @@ test("open_project_request groups a plain git worktree under an existing repo pr
const session = createSessionForWorkspaceTests(); const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>(); const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>(); const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const repoRoot = path.resolve("/Users/moboudra/dev/paseo"); const repoRoot = path.resolve("/home/developer/dev/paseo");
const cwd = path.join( const cwd = path.join(
path.resolve("/Users/moboudra"), path.resolve("/home/developer"),
".paseo", ".paseo",
"worktrees", "worktrees",
"orchestrate", "orchestrate",
@@ -12,7 +12,7 @@ import {
export type DaemonClientConfig = Omit< export type DaemonClientConfig = Omit<
SharedDaemonClientConfig, SharedDaemonClientConfig,
"webSocketFactory" | "transportFactory" | "clientId" "webSocketFactory" | "transportFactory" | "clientId"
>; > & { clientId?: string };
export type CreateAgentOptions = CreateAgentRequestOptions; export type CreateAgentOptions = CreateAgentRequestOptions;
export { type SendMessageOptions, type DaemonEvent, type DaemonEventHandler }; export { type SendMessageOptions, type DaemonEvent, type DaemonEventHandler };
@@ -25,7 +25,7 @@ function nextTestClientId(): string {
export class DaemonClient extends SharedDaemonClient { export class DaemonClient extends SharedDaemonClient {
constructor(config: DaemonClientConfig) { constructor(config: DaemonClientConfig) {
const clientId = nextTestClientId(); const clientId = config.clientId ?? nextTestClientId();
super({ super({
...config, ...config,
clientId, clientId,
@@ -0,0 +1,406 @@
import { createServer, type Server as HTTPServer } from "node:http";
import type { AddressInfo } from "node:net";
import type {
BrowserAutomationExecuteRequest,
BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
import type pino from "pino";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { AgentManager } from "./agent/agent-manager.js";
import type { AgentStorage } from "./agent/agent-storage.js";
import { BrowserToolsBroker } from "./browser-tools/broker.js";
import { StaticBrowserToolsPolicy } from "./browser-tools/policy.js";
import type { CheckoutDiffManager } from "./checkout-diff-manager.js";
import type { FileBackedChatService } from "./chat/chat-service.js";
import type { DaemonConfigStore } from "./daemon-config-store.js";
import type { DownloadTokenStore } from "./file-download/token-store.js";
import type { LoopService } from "./loop-service.js";
import type { ScheduleService } from "./schedule/service.js";
import { createStub } from "./test-utils/class-mocks.js";
import { DaemonClient } from "./test-utils/daemon-client.js";
import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js";
import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
interface BrowserToolsDaemonHarness {
broker: BrowserToolsBroker;
connectDesktopBrowserClient(
options?: ConnectDesktopBrowserClientOptions,
): Promise<DesktopBrowserClientHandle>;
stop(): Promise<void>;
}
interface ConnectDesktopBrowserClientOptions {
clientId?: string;
capabilities?: Partial<Record<string, boolean>>;
}
interface DesktopBrowserClientHandle {
clientId: string;
nextBrowserRequest(): Promise<BrowserAutomationExecuteRequest>;
respondToBrowserRequest(response: BrowserAutomationExecuteResponse): void;
disconnect(): Promise<void>;
}
interface QueuedBrowserRequests {
next(): Promise<BrowserAutomationExecuteRequest>;
push(request: BrowserAutomationExecuteRequest): void;
close(): void;
}
const harnesses: BrowserToolsDaemonHarness[] = [];
afterEach(async () => {
await Promise.all(harnesses.splice(0).map((harness) => harness.stop()));
});
describe("WebSocketServer browser tools wiring", () => {
it("registers capable clients and dispatches broker requests over the real WebSocket path", async () => {
const harness = await startBrowserToolsDaemonHarness();
const desktop = await harness.connectDesktopBrowserClient();
const resultPromise = harness.broker.execute({
command: { command: "list_tabs", args: {} },
});
const request = await desktop.nextBrowserRequest();
expect(request).toMatchObject({
type: "browser.automation.execute.request",
requestId: "req-1",
command: { command: "list_tabs", args: {} },
});
desktop.respondToBrowserRequest({
type: "browser.automation.execute.response",
payload: {
requestId: request.requestId,
ok: true,
result: { command: "list_tabs", tabs: [] },
},
});
await expect(resultPromise).resolves.toEqual({
requestId: request.requestId,
ok: true,
result: { command: "list_tabs", tabs: [] },
});
});
it("unregisters capable clients on disconnect and clears pending browser commands", async () => {
const harness = await startBrowserToolsDaemonHarness();
const desktop = await harness.connectDesktopBrowserClient();
const pendingResult = harness.broker.execute({
command: { command: "list_tabs", args: {} },
});
const pendingExpectation = expect(pendingResult).resolves.toMatchObject({
ok: false,
error: { code: "browser_no_desktop", retryable: true },
});
await desktop.nextBrowserRequest();
expect(harness.broker.getPendingRequestCount()).toBe(1);
await desktop.disconnect();
expect(harness.broker.getRegisteredClientCount()).toBe(0);
expect(harness.broker.getPendingRequestCount()).toBe(0);
await pendingExpectation;
await expect(
harness.broker.execute({ command: { command: "list_tabs", args: {} } }),
).resolves.toMatchObject({
ok: false,
error: { code: "browser_no_desktop" },
});
});
it("keeps browser automation registered when a desktop browser client resumes", async () => {
const harness = await startBrowserToolsDaemonHarness();
const clientId = "desktop-client-1";
await harness.connectDesktopBrowserClient({
clientId,
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
});
const resumedDesktop = await harness.connectDesktopBrowserClient({
clientId,
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
});
const resultPromise = harness.broker.execute({
command: { command: "click", args: { ref: "@e1" } },
});
const request = await resumedDesktop.nextBrowserRequest();
resumedDesktop.respondToBrowserRequest({
type: "browser.automation.execute.response",
payload: {
requestId: request.requestId,
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e1" },
},
});
await expect(resultPromise).resolves.toMatchObject({
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e1" },
});
});
});
async function startBrowserToolsDaemonHarness(): Promise<BrowserToolsDaemonHarness> {
const httpServer = createServer();
const broker = createBroker();
const wsServer = createVoiceAssistantWebSocketServer({ httpServer, broker });
const clients = new Set<DaemonClient>();
await listen(httpServer);
const url = `ws://127.0.0.1:${getPort(httpServer)}/ws`;
const harness: BrowserToolsDaemonHarness = {
broker,
async connectDesktopBrowserClient(options = {}) {
const clientId = options.clientId;
const client = new DaemonClient({
url,
...(clientId ? { clientId } : {}),
clientType: "browser",
connectTimeoutMs: 500,
reconnect: { enabled: false },
capabilities: options.capabilities ?? { [CLIENT_CAPS.desktopBrowserAutomation]: true },
});
clients.add(client);
const requests = createBrowserRequestQueue();
client.on("browser.automation.execute.request", (request) => {
requests.push(request);
});
await client.connect();
return {
clientId: clientId ?? "",
nextBrowserRequest: () => requests.next(),
respondToBrowserRequest: (response) =>
client.sendBrowserAutomationExecuteResponse(response),
async disconnect() {
requests.close();
clients.delete(client);
await client.close();
await waitFor(() => broker.getRegisteredClientCount() === 0);
},
};
},
async stop() {
await Promise.all(Array.from(clients, (client) => client.close()));
clients.clear();
await wsServer.close();
await closeHttpServer(httpServer);
},
};
harnesses.push(harness);
return harness;
}
function createBroker(): BrowserToolsBroker {
return new BrowserToolsBroker({
policy: new StaticBrowserToolsPolicy(true),
defaultTimeoutMs: 500,
createRequestId: createRequestIdSequence(),
});
}
function createRequestIdSequence(): () => string {
let index = 0;
return () => {
index += 1;
return `req-${index}`;
};
}
function createVoiceAssistantWebSocketServer(params: {
httpServer: HTTPServer;
broker: BrowserToolsBroker;
}): VoiceAssistantWebSocketServer {
const { httpServer, broker } = params;
const agentManager = {
setAgentAttentionCallback: vi.fn(),
subscribe: vi.fn(() => () => {}),
getMetricsSnapshot: vi.fn(() => ({
total: 0,
byLifecycle: {},
withActiveForegroundTurn: 0,
timelineStats: { totalItems: 0, maxItemsPerAgent: 0 },
})),
};
const daemonConfigStore = {
onChange: vi.fn(() => () => {}),
};
return new VoiceAssistantWebSocketServer(
httpServer,
createStub<pino.Logger>(createLogger()),
"srv-test",
createStub<AgentManager>(agentManager),
createStub<AgentStorage>({}),
createStub<DownloadTokenStore>({}),
"/tmp/paseo-browser-tools-websocket-test",
createStub<DaemonConfigStore>(daemonConfigStore),
null,
{ allowedOrigins: new Set(["*"]) },
undefined,
undefined,
undefined,
undefined,
"1.2.3-test",
undefined,
undefined,
undefined,
createStub<FileBackedChatService>({}),
createStub<LoopService>({}),
createStub<ScheduleService>({}),
createStub<CheckoutDiffManager>({
subscribe: vi.fn(),
scheduleRefreshForCwd: vi.fn(),
getMetrics: vi.fn(() => ({
checkoutDiffTargetCount: 0,
checkoutDiffSubscriptionCount: 0,
checkoutDiffWatcherCount: 0,
checkoutDiffFallbackRefreshTargetCount: 0,
})),
dispose: vi.fn(),
}),
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
createProviderSnapshotManagerStub().manager,
undefined,
undefined,
broker,
);
}
function createBrowserRequestQueue(): QueuedBrowserRequests {
const requests: BrowserAutomationExecuteRequest[] = [];
const waiters: Array<{
resolve: (request: BrowserAutomationExecuteRequest) => void;
reject: (error: Error) => void;
}> = [];
let closed = false;
return {
next() {
const request = requests.shift();
if (request) {
return Promise.resolve(request);
}
if (closed) {
return Promise.reject(new Error("Desktop browser client disconnected"));
}
return new Promise<BrowserAutomationExecuteRequest>((resolve, reject) => {
let timeout: ReturnType<typeof setTimeout>;
const waiter = {
resolve: (value: BrowserAutomationExecuteRequest) => {
clearTimeout(timeout);
resolve(value);
},
reject: (error: Error) => {
clearTimeout(timeout);
reject(error);
},
};
timeout = setTimeout(() => {
const waiterIndex = waiters.indexOf(waiter);
if (waiterIndex !== -1) {
waiters.splice(waiterIndex, 1);
}
reject(new Error("Timed out waiting for browser automation request"));
}, 500);
waiters.push(waiter);
});
},
push(request) {
const waiter = waiters.shift();
if (waiter) {
waiter.resolve(request);
return;
}
requests.push(request);
},
close() {
closed = true;
for (const waiter of waiters.splice(0)) {
waiter.reject(new Error("Desktop browser client disconnected"));
}
},
};
}
function createLogger() {
const logger = {
child: vi.fn(() => logger),
trace: vi.fn(),
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
};
return logger;
}
function listen(server: HTTPServer): Promise<void> {
return new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", () => {
server.off("error", reject);
resolve();
});
});
}
function getPort(server: HTTPServer): number {
const address = server.address();
if (!isAddressInfo(address)) {
throw new Error("HTTP test server did not bind to a TCP port");
}
return address.port;
}
function isAddressInfo(address: string | AddressInfo | null): address is AddressInfo {
return typeof address === "object" && address !== null && typeof address.port === "number";
}
function closeHttpServer(server: HTTPServer): Promise<void> {
if (!server.listening) {
return Promise.resolve();
}
return new Promise((resolve, reject) => {
server.close((error) => {
if (error) {
reject(error);
return;
}
resolve();
});
});
}
async function waitFor(predicate: () => boolean): Promise<void> {
const startedAt = Date.now();
while (!predicate()) {
if (Date.now() - startedAt > 500) {
throw new Error("Timed out waiting for browser tools WebSocket state");
}
await new Promise((resolve) => setTimeout(resolve, 10));
}
}
@@ -69,6 +69,9 @@ import {
CLIENT_SHUTDOWN_RPC_REASON, CLIENT_SHUTDOWN_RPC_REASON,
normalizeClientRestartRpcReason, normalizeClientRestartRpcReason,
} from "./lifecycle-reasons.js"; } from "./lifecycle-reasons.js";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
import type { BrowserAutomationExecuteResponse } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { BrowserToolsBroker } from "./browser-tools/broker.js";
const WS_CLOSE_DAEMON_AUTH_FAILED = 4401; const WS_CLOSE_DAEMON_AUTH_FAILED = 4401;
@@ -304,6 +307,12 @@ function bufferFromWsData(data: Buffer | ArrayBuffer | Buffer[] | string): Buffe
return Buffer.from(data); return Buffer.from(data);
} }
function hasDesktopBrowserAutomationCapability(
capabilities: Record<string, unknown> | null,
): boolean {
return capabilities?.[CLIENT_CAPS.desktopBrowserAutomation] === true;
}
interface WebSocketLike { interface WebSocketLike {
readyState: number; readyState: number;
bufferedAmount?: number; bufferedAmount?: number;
@@ -323,6 +332,10 @@ interface SessionConnection {
externalDisconnectCleanupTimeout: ReturnType<typeof setTimeout> | null; externalDisconnectCleanupTimeout: ReturnType<typeof setTimeout> | null;
} }
interface BrowserToolsRegistration {
unregister: () => void;
}
const SLOW_REQUEST_THRESHOLD_MS = 500; const SLOW_REQUEST_THRESHOLD_MS = 500;
const EXTERNAL_SESSION_DISCONNECT_GRACE_MS = 90_000; const EXTERNAL_SESSION_DISCONNECT_GRACE_MS = 90_000;
const HELLO_TIMEOUT_MS = 15_000; const HELLO_TIMEOUT_MS = 15_000;
@@ -438,6 +451,8 @@ export class VoiceAssistantWebSocketServer {
private unsubscribeDaemonConfigChange: (() => void) | null = null; private unsubscribeDaemonConfigChange: (() => void) | null = null;
private readonly providerUsageService: ProviderUsageService; private readonly providerUsageService: ProviderUsageService;
private unsubscribeTerminalActivity: (() => void) | null = null; private unsubscribeTerminalActivity: (() => void) | null = null;
private readonly browserToolsBroker: BrowserToolsBroker | null;
private readonly browserToolsRegistrations = new Map<string, BrowserToolsRegistration>();
constructor( constructor(
server: HTTPServer, server: HTTPServer,
@@ -491,6 +506,7 @@ export class VoiceAssistantWebSocketServer {
}; };
}, },
serviceProxyPublicBaseUrl?: string | null, serviceProxyPublicBaseUrl?: string | null,
browserToolsBroker?: BrowserToolsBroker | null,
) { ) {
this.logger = logger.child({ module: "websocket-server" }); this.logger = logger.child({ module: "websocket-server" });
this.serverId = serverId; this.serverId = serverId;
@@ -499,6 +515,7 @@ export class VoiceAssistantWebSocketServer {
} }
this.daemonVersion = daemonVersion.trim(); this.daemonVersion = daemonVersion.trim();
this.daemonRuntimeConfig = daemonRuntimeConfig; this.daemonRuntimeConfig = daemonRuntimeConfig;
this.browserToolsBroker = browserToolsBroker ?? null;
this.agentManager = agentManager; this.agentManager = agentManager;
this.agentStorage = agentStorage; this.agentStorage = agentStorage;
this.projectRegistry = projectRegistry ?? createNoopProjectRegistry(); this.projectRegistry = projectRegistry ?? createNoopProjectRegistry();
@@ -842,6 +859,9 @@ export class VoiceAssistantWebSocketServer {
this.sessions.clear(); this.sessions.clear();
this.socketIdentities.clear(); this.socketIdentities.clear();
this.externalSessionsByKey.clear(); this.externalSessionsByKey.clear();
for (const clientId of this.browserToolsRegistrations.keys()) {
this.unregisterBrowserToolsClient(clientId);
}
this.wss.close(); this.wss.close();
} }
@@ -1124,10 +1144,12 @@ export class VoiceAssistantWebSocketServer {
) { ) {
existing.clientCapabilities = newClientCapabilities; existing.clientCapabilities = newClientCapabilities;
existing.session.updateClientCapabilities(newClientCapabilities); existing.session.updateClientCapabilities(newClientCapabilities);
this.syncBrowserToolsClientRegistration(existing);
} }
existing.sockets.add(ws); existing.sockets.add(ws);
this.sessions.set(ws, existing); this.sessions.set(ws, existing);
pending.identity.sessionId = existing.session.getSessionId(); pending.identity.sessionId = existing.session.getSessionId();
this.syncBrowserToolsClientRegistration(existing);
this.sendToClient(ws, this.createServerInfoMessage()); this.sendToClient(ws, this.createServerInfoMessage());
pending.connectionLogger.info( pending.connectionLogger.info(
{ {
@@ -1152,6 +1174,7 @@ export class VoiceAssistantWebSocketServer {
this.sessions.set(ws, connection); this.sessions.set(ws, connection);
this.externalSessionsByKey.set(clientId, connection); this.externalSessionsByKey.set(clientId, connection);
pending.identity.sessionId = connection.session.getSessionId(); pending.identity.sessionId = connection.session.getSessionId();
this.syncBrowserToolsClientRegistration(connection);
this.sendToClient(ws, this.createServerInfoMessage()); this.sendToClient(ws, this.createServerInfoMessage());
connection.connectionLogger.info( connection.connectionLogger.info(
{ {
@@ -1314,6 +1337,7 @@ export class VoiceAssistantWebSocketServer {
this.socketIdentities.delete(ws); this.socketIdentities.delete(ws);
if (connection.sockets.size === 0) { if (connection.sockets.size === 0) {
this.unregisterBrowserToolsClient(connection.clientId);
this.incrementRuntimeCounter("sessionDisconnectedWaitingReconnect"); this.incrementRuntimeCounter("sessionDisconnectedWaitingReconnect");
if (connection.externalDisconnectCleanupTimeout) { if (connection.externalDisconnectCleanupTimeout) {
clearTimeout(connection.externalDisconnectCleanupTimeout); clearTimeout(connection.externalDisconnectCleanupTimeout);
@@ -1375,6 +1399,7 @@ export class VoiceAssistantWebSocketServer {
if (existing === connection) { if (existing === connection) {
this.externalSessionsByKey.delete(connection.clientId); this.externalSessionsByKey.delete(connection.clientId);
} }
this.unregisterBrowserToolsClient(connection.clientId);
connection.connectionLogger.trace( connection.connectionLogger.trace(
{ clientId: connection.clientId, totalSessions: this.sessions.size }, { clientId: connection.clientId, totalSessions: this.sessions.size },
@@ -1383,6 +1408,39 @@ export class VoiceAssistantWebSocketServer {
await connection.session.cleanup(); await connection.session.cleanup();
} }
private syncBrowserToolsClientRegistration(connection: SessionConnection): void {
if (!this.browserToolsBroker) {
return;
}
if (!hasDesktopBrowserAutomationCapability(connection.clientCapabilities)) {
this.unregisterBrowserToolsClient(connection.clientId);
return;
}
const existing = this.browserToolsRegistrations.get(connection.clientId);
if (existing) {
return;
}
const unregister = this.browserToolsBroker.registerClient({
id: connection.clientId,
sendBrowserAutomationRequest: (request) => {
this.sendToConnection(connection, wrapSessionMessage(request));
},
});
this.browserToolsRegistrations.set(connection.clientId, {
unregister,
});
}
private unregisterBrowserToolsClient(clientId: string): void {
const registration = this.browserToolsRegistrations.get(clientId);
if (!registration) {
return;
}
this.browserToolsRegistrations.delete(clientId);
registration.unregister();
}
private handleInvalidInboundMessage(args: { private handleInvalidInboundMessage(args: {
ws: WebSocketLike; ws: WebSocketLike;
parsed: unknown; parsed: unknown;
@@ -1625,6 +1683,11 @@ export class VoiceAssistantWebSocketServer {
"ws_control_rpc_received", "ws_control_rpc_received",
); );
} }
if (message.message.type === "browser.automation.execute.response") {
this.browserToolsBroker?.receiveResponse(message.message as BrowserAutomationExecuteResponse);
return;
}
const startMs = performance.now(); const startMs = performance.now();
await activeConnection.session.handleMessage(message.message); await activeConnection.session.handleMessage(message.message);
const durationMs = performance.now() - startMs; const durationMs = performance.now() - startMs;
+6 -3
View File
@@ -12,19 +12,22 @@
"scripts": { "scripts": {
"daemon": { "daemon": {
"type": "service", "type": "service",
"command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"$PWD\" PASEO_HOME=\"$PWD/.dev/paseo-home\" PASEO_SKIP_DEV_SERVER_BUILD=1 PASEO_LISTEN=0.0.0.0:$PASEO_PORT ./scripts/dev-daemon.sh" "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"${PASEO_WORKTREE_PATH:-$PWD}\" PASEO_HOME=\"${PASEO_WORKTREE_PATH:-$PWD}/.dev/paseo-home\" PASEO_SKIP_DEV_SERVER_BUILD=1 PASEO_LISTEN=0.0.0.0:${PASEO_PORT:-6768} ./scripts/dev-daemon.sh"
}, },
"app": { "app": {
"type": "service", "type": "service",
"command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"$PWD\" PASEO_HOME=\"$PWD/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT} EXPO_PORT=$PASEO_PORT ./scripts/dev-app.sh" "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"${PASEO_WORKTREE_PATH:-$PWD}\" PASEO_HOME=\"${PASEO_WORKTREE_PATH:-$PWD}/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT:-6768} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT:-6768} EXPO_PORT=${PASEO_PORT:-} ./scripts/dev-app.sh"
}, },
"desktop": { "desktop": {
"type": "service", "type": "service",
"command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"$PWD\" PASEO_HOME=\"$PWD/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT} EXPO_PORT=$PASEO_PORT npm run dev --workspace=@getpaseo/desktop" "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"${PASEO_WORKTREE_PATH:-$PWD}\" PASEO_HOME=\"${PASEO_WORKTREE_PATH:-$PWD}/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT:-6768} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT:-6768} EXPO_PORT=${PASEO_PORT:-} npm run dev --workspace=@getpaseo/desktop"
}, },
"ios-simulator": { "ios-simulator": {
"type": "service", "type": "service",
"command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"$PWD\" PASEO_HOME=\"$PWD/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT} ./scripts/paseo-ios-simulator-service.sh" "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"$PWD\" PASEO_HOME=\"$PWD/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT} ./scripts/paseo-ios-simulator-service.sh"
},
"typecheck": {
"command": "npm run typecheck"
} }
} }
} }