Add opt-in browser tools for desktop tabs

* Add opt-in browser tools for desktop tabs

Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs.

* Fix browser tools CI expectations

* Address browser tools review findings

* Restrict browser file automation paths

* Fix browser upload test on Windows

* Harden browser navigation inputs

* Make browser tools create usable tabs

* Update browser MCP empty-state test

* Fail browser tab creation when registration times out

* Fix browser screenshots for agents

* Hide disabled browser tools from agents

* Address browser tools architecture review

* Replace browser tools review tests

* Wrap browser tab registration errors

* Mock Expo Router in app unit tests

* Handle invalid browser automation requests

* Return browser failure on desktop disconnect

* Update browser disconnect websocket test

* Relax browser timeout polling test

* Handle invalid browser responses

* Return browser failure when send fails

* Remove local diagnostics and fixture paths

* Fix dev service home fallback

* Use worktree home for dev services

* Use managed daemon in desktop dev

* fix(browser): keep agent tabs addressable

Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution.

* refactor(browser): register tools through catalog

Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter.

* fix(settings): translate browser tools host error
This commit is contained in:
Mohamed Boudra
2026-06-30 22:33:18 +02:00
committed by GitHub
parent db03b1f3fd
commit 20385bdb50
71 changed files with 10908 additions and 115 deletions
@@ -578,6 +578,91 @@ function createPaseoWorktreeForMcpTest(options: {
};
}
describe("browser MCP tools", () => {
const logger = createTestLogger();
it("keeps browser tools registered when browser tools are disabled", async () => {
const { agentManager, agentStorage, spies } = createTestDeps();
spies.agentManager.getAgent.mockReturnValue({ id: "agent-1", cwd: REPO_CWD });
const execute = vi.fn().mockResolvedValue({
requestId: "req-browser-disabled",
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled.",
retryable: false,
},
});
const server = await createAgentMcpServer({
agentManager,
agentStorage,
providerSnapshotManager: createOpenCodeManager().manager,
browserToolsBroker: { execute } as never,
callerAgentId: "agent-1",
logger,
});
const tool = registeredTool(server, "browser_list_tabs");
const response = await tool.handler({});
expect(lookupTool(server, "browser_page_info")).not.toBeUndefined();
expect(execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: REPO_CWD,
workspaceId: REPO_CWD,
command: { command: "list_tabs", args: { workspaceId: REPO_CWD } },
});
expect(response.structuredContent).toEqual({
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled.",
retryable: false,
},
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD },
});
});
it("wires browser tools through the browser tools broker", async () => {
const { agentManager, agentStorage, spies } = createTestDeps();
spies.agentManager.getAgent.mockReturnValue({ id: "agent-1", cwd: REPO_CWD });
const execute = vi.fn().mockResolvedValue({
requestId: "req-browser-tabs",
ok: true,
result: { command: "list_tabs", tabs: [] },
});
const server = await createAgentMcpServer({
agentManager,
agentStorage,
providerSnapshotManager: createOpenCodeManager().manager,
browserToolsBroker: { execute } as never,
callerAgentId: "agent-1",
logger,
});
const tool = registeredTool(server, "browser_list_tabs");
const response = await tool.handler({});
expect(execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: REPO_CWD,
workspaceId: REPO_CWD,
command: { command: "list_tabs", args: { workspaceId: REPO_CWD } },
});
expect(response.content).toEqual([
{
type: "text",
text: "No Paseo browser tabs are open. Call browser_new_tab to create one, then use the returned browserId or omit browserId for the active tab.",
},
]);
expect(response.structuredContent).toEqual({
ok: true,
result: { command: "list_tabs", tabs: [] },
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD },
});
});
});
describe("terminal MCP tools", () => {
const logger = createTestLogger();
@@ -280,7 +280,7 @@ describe("claude tool-call mapper", () => {
name: "Grep",
input: {
pattern: '\\\\\\"cli\\\\\\""',
path: "/Users/moboudra/dev/paseo/packages/desktop/src",
path: "/workspaces/paseo/packages/desktop/src",
output_mode: "content",
"-n": true,
},
@@ -188,9 +188,9 @@ describe("opencode tool-call mapper", () => {
toolName: "read",
callId: "opencode-read-xml",
status: "completed",
input: { filePath: "/Users/moboudra/dev/paseo/docs/release.md" },
input: { filePath: "/workspaces/paseo/docs/release.md" },
output: [
"<path>/Users/moboudra/dev/paseo/docs/release.md</path>",
"<path>/workspaces/paseo/docs/release.md</path>",
"<type>file</type>",
"<content>",
"1: # Release",
@@ -203,7 +203,7 @@ describe("opencode tool-call mapper", () => {
expect(item.detail).toEqual({
type: "read",
filePath: "/Users/moboudra/dev/paseo/docs/release.md",
filePath: "/workspaces/paseo/docs/release.md",
content: [
"1: # Release",
"2:",
@@ -282,8 +282,7 @@ describe("opencode tool-call mapper", () => {
callId: "opencode-write-success-text",
status: "completed",
input: {
filePath:
"/Users/moboudra/dev/paseo/.dev/paseo-home/worktrees/1luy0po7/cold-ladybug/dummy.txt",
filePath: "/workspaces/paseo/worktrees/cold-ladybug/dummy.txt",
content: "hello world\n",
},
output: "Wrote file successfully.",
@@ -292,8 +291,7 @@ describe("opencode tool-call mapper", () => {
expect(item.detail).toEqual({
type: "write",
filePath:
"/Users/moboudra/dev/paseo/.dev/paseo-home/worktrees/1luy0po7/cold-ladybug/dummy.txt",
filePath: "/workspaces/paseo/worktrees/cold-ladybug/dummy.txt",
content: "hello world\n",
});
});
@@ -305,9 +303,9 @@ describe("opencode tool-call mapper", () => {
callId: "opencode-edit-camel",
status: "completed",
input: {
filePath: "/Users/moboudra/dev/paseo/packages/website/src/data/agent-pages.ts",
oldString: 'metaTitle: "Junie agent Mobile and Desktop App, Open Source"',
newString: 'metaTitle: "Junie Agent Mobile and Desktop App, Open Source"',
filePath: "/workspaces/paseo/packages/website/src/data/agent-pages.ts",
oldString: 'metaTitle: "Agent page"',
newString: 'metaTitle: "Updated agent page"',
},
output: "Edit applied successfully.",
}),
@@ -315,9 +313,9 @@ describe("opencode tool-call mapper", () => {
expect(item.detail).toEqual({
type: "edit",
filePath: "/Users/moboudra/dev/paseo/packages/website/src/data/agent-pages.ts",
oldString: 'metaTitle: "Junie agent Mobile and Desktop App, Open Source"',
newString: 'metaTitle: "Junie Agent Mobile and Desktop App, Open Source"',
filePath: "/workspaces/paseo/packages/website/src/data/agent-pages.ts",
oldString: 'metaTitle: "Agent page"',
newString: 'metaTitle: "Updated agent page"',
});
});
@@ -73,6 +73,8 @@ import {
type CreatePaseoWorktreeCommandInput,
listPaseoWorktreesCommand,
} from "../../worktree/commands.js";
import { registerBrowserTools } from "../../browser-tools/tools.js";
import type { BrowserToolsBroker } from "../../browser-tools/broker.js";
import type {
PaseoToolCatalog,
PaseoToolConfig,
@@ -102,6 +104,7 @@ export interface PaseoToolHostDependencies {
createPaseoWorktree?: CreatePaseoWorktreeWorkflowFn;
// Mints a fresh directory workspace for a cwd and returns its id.
ensureWorkspaceForCreate?: (cwd: string) => Promise<string>;
browserToolsBroker?: BrowserToolsBroker | null;
paseoHome?: string;
worktreesRoot?: string;
/**
@@ -1002,6 +1005,15 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
return toCatalog();
}
if (options.browserToolsBroker) {
registerBrowserTools({
registerTool,
broker: options.browserToolsBroker,
callerAgentId,
resolveCallerAgent,
});
}
registerTool(
"create_agent",
{
+46 -22
View File
@@ -118,7 +118,9 @@ import { FileBackedChatService } from "./chat/chat-service.js";
import { CheckoutDiffManager } from "./checkout-diff-manager.js";
import { LoopService } from "./loop-service.js";
import { ScheduleService } from "./schedule/service.js";
import { DaemonConfigStore } from "./daemon-config-store.js";
import { DaemonConfigStore, type MutableDaemonConfig } from "./daemon-config-store.js";
import { BrowserToolsBroker } from "./browser-tools/broker.js";
import { DaemonConfigBrowserToolsPolicy } from "./browser-tools/policy.js";
import { WorkspaceGitServiceImpl } from "./workspace-git-service.js";
import { resolveWorkspaceIdForPath } from "./resolve-workspace-id-for-path.js";
import {
@@ -330,6 +332,7 @@ export interface PaseoDaemonConfig {
trustedProxies?: true | string[];
mcpEnabled?: boolean;
mcpInjectIntoAgents?: boolean;
browserToolsEnabled?: boolean;
autoArchiveAfterMerge?: boolean;
enableTerminalAgentHooks?: boolean;
appendSystemPrompt?: string;
@@ -383,6 +386,7 @@ export interface PaseoDaemon {
terminalManager: TerminalManager;
serviceProxy: ServiceProxySubsystem;
scriptRuntimeStore: WorkspaceScriptRuntimeStore;
browserToolsBroker: BrowserToolsBroker;
start(): Promise<void>;
stop(): Promise<void>;
getListenTarget(): ListenTarget | null;
@@ -429,6 +433,39 @@ function resolveExpressTrustProxySetting(config: PaseoDaemonConfig): true | stri
return config.trustedProxies ?? ["loopback"];
}
function createInitialMutableDaemonConfig(config: PaseoDaemonConfig): MutableDaemonConfig {
const providers: MutableDaemonConfig["providers"] = Object.fromEntries(
Object.entries(config.providerOverrides ?? {}).map(([providerId, override]) => {
const providerConfig: MutableDaemonConfig["providers"][string] = {};
if (override.enabled !== undefined) {
providerConfig.enabled = override.enabled;
}
if (override.additionalModels) {
providerConfig.additionalModels = override.additionalModels;
}
return [providerId, providerConfig];
}),
);
const initialConfig: MutableDaemonConfig = {
mcp: { injectIntoAgents: config.mcpInjectIntoAgents ?? true },
browserTools: { enabled: config.browserToolsEnabled ?? false },
providers,
metadataGeneration: {
providers: config.metadataGeneration?.providers ?? [],
},
autoArchiveAfterMerge: config.autoArchiveAfterMerge ?? false,
enableTerminalAgentHooks: config.enableTerminalAgentHooks ?? false,
appendSystemPrompt: config.appendSystemPrompt ?? "",
};
if (config.terminalProfiles !== undefined) {
initialConfig.terminalProfiles = config.terminalProfiles;
}
return initialConfig;
}
export async function createPaseoDaemon(
config: PaseoDaemonConfig,
rootLogger: Logger,
@@ -439,29 +476,13 @@ export async function createPaseoDaemon(
const daemonVersion = resolveDaemonVersion(import.meta.url);
const daemonConfigStore = new DaemonConfigStore(
config.paseoHome,
{
mcp: { injectIntoAgents: config.mcpInjectIntoAgents ?? true },
providers: Object.fromEntries(
Object.entries(config.providerOverrides ?? {}).map(([providerId, override]) => [
providerId,
{
...(override.enabled !== undefined ? { enabled: override.enabled } : {}),
...(override.additionalModels ? { additionalModels: override.additionalModels } : {}),
},
]),
),
metadataGeneration: {
providers: config.metadataGeneration?.providers ?? [],
},
autoArchiveAfterMerge: config.autoArchiveAfterMerge ?? false,
enableTerminalAgentHooks: config.enableTerminalAgentHooks ?? false,
appendSystemPrompt: config.appendSystemPrompt ?? "",
...(config.terminalProfiles !== undefined
? { terminalProfiles: config.terminalProfiles }
: {}),
},
createInitialMutableDaemonConfig(config),
logger,
);
const browserToolsPolicy = new DaemonConfigBrowserToolsPolicy(daemonConfigStore);
const browserToolsBroker = new BrowserToolsBroker({
policy: browserToolsPolicy,
});
const serverId = getOrCreateServerId(config.paseoHome, { logger });
const daemonKeyPair = await loadOrCreateDaemonKeyPair(config.paseoHome, logger);
@@ -959,6 +980,7 @@ export async function createPaseoDaemon(
clearWorkspaceArchiving: clearWorkspaceArchivingExternal,
ensureWorkspaceForCreate: ensureWorkspaceForCreateExternal,
createPaseoWorktree: createPaseoWorktreeForTools,
browserToolsBroker,
paseoHome: config.paseoHome,
worktreesRoot: config.worktreesRoot,
callerAgentId: runtime.callerAgentId,
@@ -1228,6 +1250,7 @@ export async function createPaseoDaemon(
},
},
serviceProxyPublicBaseUrl,
browserToolsBroker,
);
if (relayEnabled) {
@@ -1327,6 +1350,7 @@ export async function createPaseoDaemon(
terminalManager,
serviceProxy,
scriptRuntimeStore,
browserToolsBroker,
start,
stop,
getListenTarget: () => boundListenTarget,
@@ -0,0 +1,330 @@
import { afterEach, describe, expect, test, vi } from "vitest";
import type {
BrowserAutomationCommand,
BrowserAutomationExecuteRequest,
BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { BrowserToolsBroker, type BrowserToolsDesktopClient } from "./broker.js";
import { StaticBrowserToolsPolicy } from "./policy.js";
class FakeDesktopClient implements BrowserToolsDesktopClient {
public readonly receivedRequests: BrowserAutomationExecuteRequest[] = [];
public constructor(public readonly id: string) {}
public sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void {
this.receivedRequests.push(request);
}
public resolveLatestWith(
broker: BrowserToolsBroker,
responsePayload: BrowserAutomationExecuteResponse["payload"],
): boolean {
return broker.receiveResponse({
type: "browser.automation.execute.response",
payload: responsePayload,
});
}
}
class FailingDesktopClient implements BrowserToolsDesktopClient {
public readonly id = "desktop-1";
public sendBrowserAutomationRequest(): void {
throw new Error("websocket send failed");
}
}
function createBroker(options: { enabled: boolean; timeoutMs?: number }): BrowserToolsBroker {
return new BrowserToolsBroker({
policy: new StaticBrowserToolsPolicy(options.enabled),
defaultTimeoutMs: options.timeoutMs ?? 100,
createRequestId: () => "req-1",
});
}
function pageInfoCommand(): BrowserAutomationCommand {
return { command: "page_info", args: { browserId: "browser-1" } };
}
describe("BrowserToolsBroker", () => {
afterEach(() => {
vi.useRealTimers();
});
test("disabled returns browser_disabled", async () => {
const broker = createBroker({ enabled: false });
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.",
retryable: false,
},
});
});
test("no capable desktop returns browser_no_desktop", async () => {
const broker = createBroker({ enabled: true });
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
retryable: true,
},
});
});
test("invalid browser requests return structured failures without contacting desktop", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
await expect(
broker.execute({
command: {
command: "new_tab",
args: { url: "ftp://example.com" },
} as unknown as BrowserAutomationCommand,
}),
).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_unknown_error",
message: "Browser automation request is invalid: URL must use http or https.",
retryable: false,
},
});
expect(client.receivedRequests).toEqual([]);
expect(broker.getPendingRequestCount()).toBe(0);
});
test("capable fake desktop receives request and returns response", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
workspaceId: "workspace-1",
});
expect(client.receivedRequests).toEqual([
{
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
},
]);
expect(broker.getPendingRequestCount()).toBe(1);
expect(
client.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
},
],
},
}),
).toBe(true);
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
isActive: false,
isLoading: false,
},
],
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("desktop receives snapshot requests", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
workspaceId: "workspace-1",
});
expect(client.receivedRequests).toEqual([
{
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
},
]);
client.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
elements: [],
},
});
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
elements: [],
},
});
});
test("timeout resolves browser_timeout and clears pending state", async () => {
vi.useFakeTimers();
const broker = createBroker({ enabled: true, timeoutMs: 50 });
broker.registerClient(new FakeDesktopClient("desktop-1"));
const resultPromise = broker.execute({ command: pageInfoCommand() });
expect(broker.getPendingRequestCount()).toBe(1);
await vi.advanceTimersByTimeAsync(50);
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_timeout",
message: "Browser automation timed out after 50ms.",
retryable: true,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("disconnect resolves retryable failure and clears pending request", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
const unregister = broker.registerClient(client);
const resultPromise = broker.execute({ command: pageInfoCommand() });
expect(broker.getPendingRequestCount()).toBe(1);
unregister();
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_no_desktop",
message: "The desktop browser automation client disconnected before responding.",
retryable: true,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("desktop send failure resolves structured failure and clears pending request", async () => {
const broker = createBroker({ enabled: true });
broker.registerClient(new FailingDesktopClient());
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_unknown_error",
message: "Browser automation request failed to send: websocket send failed",
retryable: false,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("explicit browser failure response propagates typed error", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({ command: pageInfoCommand() });
client.resolveLatestWith(broker, {
requestId: "req-1",
ok: false,
error: {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
},
});
await expect(resultPromise).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("invalid browser response resolves a structured failure and clears pending state", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({ command: pageInfoCommand() });
expect(broker.getPendingRequestCount()).toBe(1);
expect(
broker.receiveResponse({
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: true,
result: { command: "future_command" },
},
} as unknown as BrowserAutomationExecuteResponse),
).toBe(true);
await expect(resultPromise).resolves.toMatchObject({
requestId: "req-1",
ok: false,
error: {
code: "browser_unknown_error",
retryable: false,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
});
@@ -0,0 +1,284 @@
import { randomUUID } from "node:crypto";
import {
BrowserAutomationExecuteRequestSchema,
BrowserAutomationExecuteResponseSchema,
type BrowserAutomationCommand,
type BrowserAutomationExecuteRequest,
type BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { browserToolsFailure, type BrowserToolsResponsePayload } from "./errors.js";
import type { BrowserToolsPolicy } from "./policy.js";
export interface BrowserToolsDesktopClient {
id: string;
sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void | Promise<void>;
}
export interface BrowserToolsExecuteInput {
command: BrowserAutomationCommand;
agentId?: string;
cwd?: string;
workspaceId?: string;
browserId?: string;
requestId?: string;
timeoutMs?: number;
}
interface PendingBrowserToolsRequest {
clientId: string;
timeout: ReturnType<typeof setTimeout>;
resolve: (payload: BrowserToolsResponsePayload) => void;
}
export interface BrowserToolsBrokerOptions {
policy: BrowserToolsPolicy;
defaultTimeoutMs?: number;
createRequestId?: () => string;
}
const DEFAULT_BROWSER_TOOLS_TIMEOUT_MS = 15_000;
export class BrowserToolsBroker {
private readonly policy: BrowserToolsPolicy;
private readonly defaultTimeoutMs: number;
private readonly createRequestId: () => string;
private readonly clients = new Map<string, BrowserToolsDesktopClient>();
private readonly pending = new Map<string, PendingBrowserToolsRequest>();
public constructor(options: BrowserToolsBrokerOptions) {
this.policy = options.policy;
this.defaultTimeoutMs = options.defaultTimeoutMs ?? DEFAULT_BROWSER_TOOLS_TIMEOUT_MS;
this.createRequestId = options.createRequestId ?? (() => `browser_${randomUUID()}`);
}
public registerClient(client: BrowserToolsDesktopClient): () => void {
this.clients.set(client.id, client);
return () => this.unregisterClient(client.id);
}
public unregisterClient(clientId: string): void {
const deleted = this.clients.delete(clientId);
if (!deleted) {
return;
}
for (const [requestId, pending] of this.pending) {
if (pending.clientId !== clientId) {
continue;
}
this.pending.delete(requestId);
clearTimeout(pending.timeout);
pending.resolve(
browserToolsFailure({
requestId,
code: "browser_no_desktop",
message: "The desktop browser automation client disconnected before responding.",
retryable: true,
}),
);
}
}
public getPendingRequestCount(): number {
return this.pending.size;
}
public getRegisteredClientCount(): number {
return this.clients.size;
}
public async execute(input: BrowserToolsExecuteInput): Promise<BrowserToolsResponsePayload> {
const requestId = input.requestId ?? this.createRequestId();
if (!this.policy.isEnabled()) {
return browserToolsFailure({
requestId,
code: "browser_disabled",
message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.",
});
}
const client = this.selectClient();
if (!client) {
return browserToolsFailure({
requestId,
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
retryable: true,
});
}
const request = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId,
...(input.agentId ? { agentId: input.agentId } : {}),
...(input.cwd ? { cwd: input.cwd } : {}),
...(input.workspaceId ? { workspaceId: input.workspaceId } : {}),
...(input.browserId ? { browserId: input.browserId } : {}),
command: input.command,
});
if (!request.success) {
return browserToolsFailure({
requestId,
code: "browser_unknown_error",
message: formatBrowserAutomationValidationError(request.error.issues[0]?.message),
});
}
return this.sendRequest({
client,
request: request.data,
timeoutMs: input.timeoutMs ?? this.defaultTimeoutMs,
});
}
public receiveResponse(response: BrowserAutomationExecuteResponse): boolean {
const parsed = BrowserAutomationExecuteResponseSchema.safeParse(response);
if (!parsed.success) {
const requestId = getBrowserAutomationResponseRequestId(response);
if (!requestId) {
return false;
}
const pending = this.pending.get(requestId);
if (!pending) {
return false;
}
this.pending.delete(requestId);
clearTimeout(pending.timeout);
pending.resolve(
browserToolsFailure({
requestId,
code: "browser_unknown_error",
message: formatBrowserAutomationResponseValidationError(parsed.error.issues[0]?.message),
}),
);
return true;
}
const pending = this.pending.get(parsed.data.payload.requestId);
if (!pending) {
return false;
}
this.pending.delete(parsed.data.payload.requestId);
clearTimeout(pending.timeout);
pending.resolve(parsed.data.payload);
return true;
}
private selectClient(): BrowserToolsDesktopClient | null {
for (const client of this.clients.values()) {
return client;
}
return null;
}
private sendRequest(params: {
client: BrowserToolsDesktopClient;
request: BrowserAutomationExecuteRequest;
timeoutMs: number;
}): Promise<BrowserToolsResponsePayload> {
const { client, request, timeoutMs } = params;
return new Promise<BrowserToolsResponsePayload>((resolve) => {
const timeout = setTimeout(() => {
if (!this.pending.delete(request.requestId)) {
return;
}
resolve(
browserToolsFailure({
requestId: request.requestId,
code: "browser_timeout",
message: `Browser automation timed out after ${timeoutMs}ms.`,
retryable: true,
}),
);
}, timeoutMs);
this.pending.set(request.requestId, {
clientId: client.id,
timeout,
resolve,
});
try {
Promise.resolve(client.sendBrowserAutomationRequest(request)).catch((error: unknown) => {
resolveSendFailure({
requestId: request.requestId,
pending: this.pending,
timeout,
resolve,
error,
});
});
} catch (error) {
resolveSendFailure({
requestId: request.requestId,
pending: this.pending,
timeout,
resolve,
error,
});
}
});
}
}
function resolveSendFailure(params: {
requestId: string;
pending: Map<string, PendingBrowserToolsRequest>;
timeout: ReturnType<typeof setTimeout>;
resolve: (payload: BrowserToolsResponsePayload) => void;
error: unknown;
}): void {
if (!params.pending.delete(params.requestId)) {
return;
}
clearTimeout(params.timeout);
params.resolve(
browserToolsFailure({
requestId: params.requestId,
code: "browser_unknown_error",
message: formatBrowserAutomationSendError(params.error),
}),
);
}
function formatBrowserAutomationValidationError(message: string | undefined): string {
if (!message) {
return "Browser automation request is invalid.";
}
return `Browser automation request is invalid: ${message}.`;
}
function formatBrowserAutomationResponseValidationError(message: string | undefined): string {
if (!message) {
return "Browser automation response is invalid.";
}
return `Browser automation response is invalid: ${message}.`;
}
function formatBrowserAutomationSendError(error: unknown): string {
if (error instanceof Error && error.message) {
return `Browser automation request failed to send: ${error.message}`;
}
return `Browser automation request failed to send: ${String(error)}`;
}
function getBrowserAutomationResponseRequestId(response: unknown): string | null {
if (!isRecord(response)) {
return null;
}
const payload = response.payload;
if (!isRecord(payload) || typeof payload.requestId !== "string") {
return null;
}
return payload.requestId;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
@@ -0,0 +1,48 @@
import type {
BrowserAutomationErrorCode,
BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
export type BrowserToolsResponsePayload = BrowserAutomationExecuteResponse["payload"];
export type BrowserToolsErrorPayload = Extract<BrowserToolsResponsePayload, { ok: false }>["error"];
export class BrowserToolsRequestError extends Error {
public readonly code: BrowserAutomationErrorCode;
public readonly retryable: boolean;
public constructor(error: BrowserToolsErrorPayload) {
super(error.message);
this.name = "BrowserToolsRequestError";
this.code = error.code;
this.retryable = error.retryable;
}
}
export function browserToolsFailure(params: {
requestId: string;
code: BrowserAutomationErrorCode;
message: string;
retryable?: boolean;
}): BrowserToolsResponsePayload {
return {
requestId: params.requestId,
ok: false,
error: {
code: params.code,
message: params.message,
retryable: params.retryable ?? false,
},
};
}
export function createBrowserToolsRequestError(params: {
code: BrowserAutomationErrorCode;
message: string;
retryable?: boolean;
}): BrowserToolsRequestError {
return new BrowserToolsRequestError({
code: params.code,
message: params.message,
retryable: params.retryable ?? false,
});
}
@@ -0,0 +1,29 @@
import type { DaemonConfigStore, MutableDaemonConfig } from "../daemon-config-store.js";
export interface BrowserToolsPolicy {
isEnabled(): boolean;
}
export class StaticBrowserToolsPolicy implements BrowserToolsPolicy {
public constructor(private readonly enabled: boolean) {}
public isEnabled(): boolean {
return this.enabled;
}
}
export class DaemonConfigBrowserToolsPolicy implements BrowserToolsPolicy {
public constructor(private readonly configStore: Pick<DaemonConfigStore, "get">) {}
public isEnabled(): boolean {
return readBrowserToolsEnabled(this.configStore.get());
}
}
function readBrowserToolsEnabled(config: MutableDaemonConfig): boolean {
const browserTools = config.browserTools;
if (typeof browserTools !== "object" || browserTools === null || Array.isArray(browserTools)) {
return false;
}
return browserTools.enabled === true;
}
@@ -0,0 +1,850 @@
import { describe, expect, it, vi } from "vitest";
import type { BrowserToolsBroker } from "./broker.js";
import type { BrowserToolsResponsePayload } from "./errors.js";
import { registerBrowserTools, type RegisterBrowserToolsOptions } from "./tools.js";
interface RegisteredTool {
config: { inputSchema: Record<string, unknown>; outputSchema: unknown };
handler: (args: Record<string, unknown>) => Promise<{
content: Array<{ type: string; text?: string }>;
structuredContent?: Record<string, unknown>;
}>;
}
function createHarness(options?: { brokerResponse?: BrowserToolsResponsePayload }) {
const tools = new Map<string, RegisteredTool>();
const execute = vi.fn(async () => {
return (
options?.brokerResponse ?? {
requestId: "req-1",
ok: true as const,
result: {
command: "list_tabs" as const,
tabs: [
{
browserId: "browser-1",
url: "https://example.com",
title: "Example",
isActive: true,
isLoading: false,
},
],
},
}
);
});
const registerTool = vi.fn((name, config, handler) => {
tools.set(name, {
config,
handler: handler as RegisteredTool["handler"],
});
}) as unknown as RegisterBrowserToolsOptions["registerTool"];
registerBrowserTools({
registerTool,
broker: { execute } as unknown as BrowserToolsBroker,
callerAgentId: "agent-1",
resolveCallerAgent: () => ({ id: "agent-1", cwd: "/repo" }),
});
return { tools, execute };
}
function tool(harness: ReturnType<typeof createHarness>, name: string): RegisteredTool {
const registered = harness.tools.get(name);
if (!registered) {
throw new Error(`Tool not registered: ${name}`);
}
return registered;
}
describe("registerBrowserTools", () => {
it("registers browser_list_tabs and routes through the broker with caller workspace", async () => {
const harness = createHarness();
const response = await tool(harness, "browser_list_tabs").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "list_tabs", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([
{
type: "text",
text: "Found 1 Paseo browser tab. Use these browserId values exactly; do not use 'default'. You may omit browserId to use the active tab.\n- browserId=browser-1 active title=\"Example\" url=https://example.com",
},
]);
expect(response.structuredContent).toEqual({
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: "browser-1",
url: "https://example.com",
title: "Example",
isActive: true,
isLoading: false,
},
],
},
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("registers browser_new_tab and returns the created tab handle", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-new-tab",
ok: true,
result: {
command: "new_tab",
browserId: "browser-new",
workspaceId: "/repo",
url: "https://example.com",
},
},
});
const response = await tool(harness, "browser_new_tab").handler({ url: "https://example.com" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "new_tab",
args: { workspaceId: "/repo", url: "https://example.com" },
},
});
expect(response.content).toEqual([
{
type: "text",
text: "Created browser tab browserId=browser-new url=https://example.com. Use this browserId exactly, or omit browserId to use this active tab.",
},
]);
});
it("routes browser_page_info through the broker with explicit browserId", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-2",
ok: true,
result: {
command: "page_info",
tab: {
browserId: "browser-2",
url: "https://example.com/docs",
title: "Docs",
isActive: false,
isLoading: false,
},
},
},
});
const response = await tool(harness, "browser_page_info").handler({ browserId: "browser-2" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
browserId: "browser-2",
command: {
command: "page_info",
args: { workspaceId: "/repo", browserId: "browser-2" },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Current page browserId=browser-2: Docs — https://example.com/docs" },
]);
expect(response.structuredContent).toEqual({
ok: true,
result: {
command: "page_info",
tab: {
browserId: "browser-2",
url: "https://example.com/docs",
title: "Docs",
isActive: false,
isLoading: false,
},
},
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo", browserId: "browser-2" },
});
});
it("routes browser_snapshot through the broker with caller workspace", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-snapshot",
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "/repo",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name" },
},
],
},
},
});
const response = await tool(harness, "browser_snapshot").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "snapshot", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([{ type: "text", text: "Snapshot captured 1 element." }]);
expect(response.structuredContent).toEqual({
ok: true,
result: {
command: "snapshot",
browserId: "browser-1",
workspaceId: "/repo",
url: "https://example.com/form",
title: "Fixture",
elements: [
{
ref: "@e1",
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name" },
},
],
},
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("routes browser_set_background through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-bg",
ok: true,
result: { command: "set_background", browserId: "browser-1", color: "red" },
},
});
const response = await tool(harness, "browser_set_background").handler({ color: "red" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "set_background", args: { workspaceId: "/repo", color: "red" } },
});
expect(response.content).toEqual([
{ type: "text", text: "Set browser page background to red." },
]);
});
it("routes browser_click through the broker with a snapshot ref", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-click",
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e2" },
},
});
const response = await tool(harness, "browser_click").handler({ ref: "@e2" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "click", args: { workspaceId: "/repo", ref: "@e2" } },
});
expect(response.content).toEqual([{ type: "text", text: "Clicked browser element @e2." }]);
expect(response.structuredContent).toEqual({
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e2" },
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("routes browser_fill through the broker with a snapshot ref and value", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-fill",
ok: true,
result: { command: "fill", browserId: "browser-1", ref: "@e1" },
},
});
const response = await tool(harness, "browser_fill").handler({ ref: "@e1", value: "Ada" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "fill", args: { workspaceId: "/repo", ref: "@e1", value: "Ada" } },
});
expect(response.content).toEqual([{ type: "text", text: "Filled browser element @e1." }]);
expect(response.structuredContent).toEqual({
ok: true,
result: { command: "fill", browserId: "browser-1", ref: "@e1" },
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("routes browser_wait through the broker with text and timeout", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-wait",
ok: true,
result: { command: "wait", browserId: "browser-1", matched: "text" },
},
});
const response = await tool(harness, "browser_wait").handler({
text: "Ready",
timeoutMs: 1000,
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
timeoutMs: 2000,
command: {
command: "wait",
args: { workspaceId: "/repo", text: "Ready", timeoutMs: 1000 },
},
});
expect(response.content).toEqual([{ type: "text", text: "Browser wait matched text." }]);
expect(response.structuredContent).toEqual({
ok: true,
result: { command: "wait", browserId: "browser-1", matched: "text" },
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("routes browser_type through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-type",
ok: true,
result: { command: "type", browserId: "browser-1", ref: "@e1" },
},
});
const response = await tool(harness, "browser_type").handler({ ref: "@e1", text: "Ada" });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "type", args: { workspaceId: "/repo", ref: "@e1", text: "Ada" } },
});
expect(response.content).toEqual([{ type: "text", text: "Typed into browser element @e1." }]);
});
it("routes browser_keypress through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-keypress",
ok: true,
result: { command: "keypress", browserId: "browser-1", ref: "@e1", key: "Enter" },
},
});
const response = await tool(harness, "browser_keypress").handler({
ref: "@e1",
key: "Enter",
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "keypress",
args: { workspaceId: "/repo", ref: "@e1", key: "Enter" },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Pressed Enter on browser element @e1." },
]);
});
it("routes browser_navigate through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-nav",
ok: true,
result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" },
},
});
const response = await tool(harness, "browser_navigate").handler({
url: "https://example.com/next",
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "navigate",
args: { workspaceId: "/repo", url: "https://example.com/next" },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Navigated browser to https://example.com/next." },
]);
});
it("routes browser_back through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-back",
ok: true,
result: { command: "back", browserId: "browser-1" },
},
});
const response = await tool(harness, "browser_back").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "back", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([{ type: "text", text: "Browser back complete." }]);
});
it("routes browser_screenshot through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-shot",
ok: true,
result: {
command: "screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 100,
height: 50,
},
},
});
const response = await tool(harness, "browser_screenshot").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "screenshot", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([
{ type: "text", text: "Captured browser screenshot (100x50)." },
{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
]);
});
it("routes browser_logs through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-logs",
ok: true,
result: {
command: "logs",
browserId: "browser-1",
console: [{ level: "info", message: "ready", timestamp: 1 }],
network: [
{
url: "https://example.com/app.js",
type: "script",
startTime: 2,
duration: 3,
},
],
},
},
});
const response = await tool(harness, "browser_logs").handler({ maxEntries: 25 });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "logs", args: { workspaceId: "/repo", maxEntries: 25 } },
});
expect(response.content).toEqual([
{ type: "text", text: "Read 1 console log and 1 network entry." },
]);
});
it("routes browser_storage through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-storage",
ok: true,
result: {
command: "storage",
browserId: "browser-1",
url: "https://example.com",
cookies: [{ name: "theme", value: "dark" }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
},
});
const response = await tool(harness, "browser_storage").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "storage", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([
{ type: "text", text: "Read 1 cookie, 1 localStorage entry, and 1 sessionStorage entry." },
]);
});
it("routes browser_environment through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-environment",
ok: true,
result: {
command: "environment",
browserId: "browser-1",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
});
const response = await tool(harness, "browser_environment").handler({
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "environment",
args: {
workspaceId: "/repo",
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
},
});
expect(response.content).toEqual([
{ type: "text", text: "Browser environment viewport is 390x844." },
]);
});
it("routes browser_full_page_screenshot through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-full-page",
ok: true,
result: {
command: "full_page_screenshot",
browserId: "browser-1",
mimeType: "image/png",
dataBase64: "iVBORw0KGgo=",
width: 390,
height: 1200,
},
},
});
const response = await tool(harness, "browser_full_page_screenshot").handler({});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "full_page_screenshot", args: { workspaceId: "/repo" } },
});
expect(response.content).toEqual([
{ type: "text", text: "Captured full-page browser screenshot (390x1200)." },
{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
]);
});
it("routes browser_pdf through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-pdf",
ok: true,
result: {
command: "pdf",
browserId: "browser-1",
mimeType: "application/pdf",
dataBase64: "JVBERg==",
},
},
});
const response = await tool(harness, "browser_pdf").handler({ landscape: true });
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "pdf",
args: { workspaceId: "/repo", landscape: true, printBackground: true },
},
});
expect(response.content).toEqual([{ type: "text", text: "Exported browser page PDF." }]);
});
it("routes browser_download through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-download",
ok: true,
result: {
command: "download",
browserId: "browser-1",
url: "https://example.com/file.txt",
filePath: "/tmp/file.txt",
totalBytes: 5,
state: "completed",
},
},
});
const response = await tool(harness, "browser_download").handler({
url: "https://example.com/file.txt",
fileName: "file.txt",
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "download",
args: { workspaceId: "/repo", url: "https://example.com/file.txt", fileName: "file.txt" },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Downloaded browser file to /tmp/file.txt." },
]);
});
it("routes browser_upload through the broker", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-upload",
ok: true,
result: {
command: "upload",
browserId: "browser-1",
ref: "@e1",
filePaths: ["/tmp/file.txt"],
},
},
});
const response = await tool(harness, "browser_upload").handler({
ref: "@e1",
filePaths: ["/tmp/file.txt"],
});
expect(harness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "upload",
args: { workspaceId: "/repo", ref: "@e1", filePaths: ["/tmp/file.txt"] },
},
});
expect(response.content).toEqual([
{ type: "text", text: "Uploaded 1 file to browser element @e1." },
]);
});
it("routes browser form control tools through the broker", async () => {
const focusHarness = createHarness({
brokerResponse: {
requestId: "req-focus",
ok: true,
result: { command: "focus", browserId: "browser-1", ref: "@e1" },
},
});
const focusResponse = await tool(focusHarness, "browser_focus").handler({ ref: "@e1" });
expect(focusHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "focus", args: { workspaceId: "/repo", ref: "@e1" } },
});
expect(focusResponse.content).toEqual([{ type: "text", text: "Focused browser element @e1." }]);
const checkHarness = createHarness({
brokerResponse: {
requestId: "req-check",
ok: true,
result: { command: "check", browserId: "browser-1", ref: "@e2", checked: false },
},
});
const checkResponse = await tool(checkHarness, "browser_check").handler({
ref: "@e2",
checked: false,
});
expect(checkHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "check", args: { workspaceId: "/repo", ref: "@e2", checked: false } },
});
expect(checkResponse.content).toEqual([
{ type: "text", text: "Unchecked browser element @e2." },
]);
const selectHarness = createHarness({
brokerResponse: {
requestId: "req-select",
ok: true,
result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" },
},
});
const selectResponse = await tool(selectHarness, "browser_select").handler({
ref: "@e3",
value: "us",
});
expect(selectHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "select", args: { workspaceId: "/repo", ref: "@e3", value: "us" } },
});
expect(selectResponse.content).toEqual([
{ type: "text", text: "Selected us in browser element @e3." },
]);
const hoverHarness = createHarness({
brokerResponse: {
requestId: "req-hover",
ok: true,
result: { command: "hover", browserId: "browser-1", ref: "@e4" },
},
});
const hoverResponse = await tool(hoverHarness, "browser_hover").handler({ ref: "@e4" });
expect(hoverHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: { command: "hover", args: { workspaceId: "/repo", ref: "@e4" } },
});
expect(hoverResponse.content).toEqual([{ type: "text", text: "Hovered browser element @e4." }]);
const dragHarness = createHarness({
brokerResponse: {
requestId: "req-drag",
ok: true,
result: { command: "drag", browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" },
},
});
const dragResponse = await tool(dragHarness, "browser_drag").handler({
sourceRef: "@e4",
targetRef: "@e5",
});
expect(dragHarness.execute).toHaveBeenCalledWith({
agentId: "agent-1",
cwd: "/repo",
workspaceId: "/repo",
command: {
command: "drag",
args: { workspaceId: "/repo", sourceRef: "@e4", targetRef: "@e5" },
},
});
expect(dragResponse.content).toEqual([
{ type: "text", text: "Dragged browser element @e4 to @e5." },
]);
});
it("returns model-actionable disabled errors with structured content", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-disabled",
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.",
retryable: false,
},
},
});
const response = await tool(harness, "browser_list_tabs").handler({});
expect(response.content).toEqual([
{
type: "text",
text: "Browser tools are disabled. Enable desktop browser tools on the host, then try again.",
},
]);
expect(response.structuredContent).toEqual({
ok: false,
error: {
code: "browser_disabled",
message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.",
retryable: false,
},
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" },
});
});
it("preserves typed broker errors", async () => {
const harness = createHarness({
brokerResponse: {
requestId: "req-timeout",
ok: false,
error: {
code: "browser_timeout",
message: "Browser automation timed out after 15000ms.",
retryable: true,
},
},
});
const response = await tool(harness, "browser_page_info").handler({});
expect(response.content).toEqual([
{
type: "text",
text: "The browser did not respond before the timeout. Try again or check the desktop app.",
},
]);
expect(response.structuredContent?.error).toEqual({
code: "browser_timeout",
message: "Browser automation timed out after 15000ms.",
retryable: true,
});
});
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,38 @@
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, test } from "vitest";
import { loadConfig } from "./config.js";
const roots: string[] = [];
async function createPaseoHome(config: unknown): Promise<string> {
const root = await mkdtemp(path.join(os.tmpdir(), "paseo-config-browser-tools-"));
roots.push(root);
const paseoHome = path.join(root, ".paseo");
await mkdir(paseoHome, { recursive: true });
await writeFile(path.join(paseoHome, "config.json"), JSON.stringify(config, null, 2));
return paseoHome;
}
describe("daemon browser tools config", () => {
afterEach(async () => {
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
});
test("defaults browser tools off when config is absent", async () => {
const home = await createPaseoHome({ version: 1 });
expect(loadConfig(home, { env: {} }).browserToolsEnabled).toBe(false);
});
test("loads browser tools opt-in from persisted daemon config", async () => {
const home = await createPaseoHome({
version: 1,
daemon: { browserTools: { enabled: true } },
});
expect(loadConfig(home, { env: {} }).browserToolsEnabled).toBe(true);
});
});
+7
View File
@@ -399,6 +399,10 @@ function resolveAppendSystemPrompt(persisted: ReturnType<typeof loadPersistedCon
return persisted.daemon?.appendSystemPrompt ?? "";
}
function resolveBrowserToolsEnabled(persisted: ReturnType<typeof loadPersistedConfig>): boolean {
return persisted.daemon?.browserTools?.enabled ?? false;
}
function resolveStaticLoadConfigSettings(
env: NodeJS.ProcessEnv,
cli: CliConfigOverrides | undefined,
@@ -408,6 +412,7 @@ function resolveStaticLoadConfigSettings(
mcpEnabled: cli?.mcpEnabled ?? persisted.daemon?.mcp?.enabled ?? true,
mcpInjectIntoAgents:
cli?.mcpInjectIntoAgents ?? persisted.daemon?.mcp?.injectIntoAgents ?? false,
browserToolsEnabled: resolveBrowserToolsEnabled(persisted),
autoArchiveAfterMerge: persisted.daemon?.autoArchiveAfterMerge ?? false,
appendSystemPrompt: resolveAppendSystemPrompt(persisted),
terminalProfiles: persisted.daemon?.terminalProfiles,
@@ -435,6 +440,7 @@ export function loadConfig(
const {
mcpEnabled,
mcpInjectIntoAgents,
browserToolsEnabled,
autoArchiveAfterMerge,
appendSystemPrompt,
terminalProfiles,
@@ -472,6 +478,7 @@ export function loadConfig(
trustedProxies,
mcpEnabled,
mcpInjectIntoAgents,
browserToolsEnabled,
autoArchiveAfterMerge,
enableTerminalAgentHooks: persisted.daemon?.enableTerminalAgentHooks ?? false,
appendSystemPrompt,
@@ -94,6 +94,7 @@ describe("DaemonConfigStore", () => {
paseoHome,
{
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {},
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
@@ -126,6 +127,7 @@ describe("DaemonConfigStore", () => {
paseoHome,
{
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {},
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
@@ -143,6 +145,29 @@ describe("DaemonConfigStore", () => {
expect(persisted.daemon?.appendSystemPrompt).toBe("Prefer terse replies.");
});
test("patch persists browser tools opt-in into config.json", () => {
const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-store-"));
tempDirs.push(paseoHome);
const store = new DaemonConfigStore(
paseoHome,
{
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {},
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
appendSystemPrompt: "",
},
undefined,
);
store.patch({ browserTools: { enabled: true } });
const persisted = loadPersistedConfig(paseoHome);
expect(persisted.daemon?.browserTools).toEqual({ enabled: true });
});
test("patch persists provider additional models into config.json", () => {
const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-store-"));
tempDirs.push(paseoHome);
@@ -151,6 +176,7 @@ describe("DaemonConfigStore", () => {
paseoHome,
{
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {},
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
@@ -192,6 +218,7 @@ describe("DaemonConfigStore", () => {
paseoHome,
{
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {},
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
@@ -240,6 +267,7 @@ describe("DaemonConfigStore", () => {
paseoHome,
{
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {},
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
@@ -292,6 +320,7 @@ describe("DaemonConfigStore", () => {
paseoHome,
{
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {},
autoArchiveAfterMerge: false,
enableTerminalAgentHooks: false,
@@ -315,6 +344,7 @@ describe("DaemonConfigStore", () => {
paseoHome,
{
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers: {},
autoArchiveAfterMerge: false,
enableTerminalAgentHooks: false,
@@ -172,6 +172,7 @@ function mergeMutableConfigIntoPersistedConfig(params: {
mutable: MutableDaemonConfig;
}): PersistedConfig {
const { persisted, mutable } = params;
const browserToolsEnabled = readBrowserToolsEnabled(mutable);
const metadataGenerationProviders = readMetadataGenerationProviders(mutable);
const providerOverrides = applyMutableProviderConfigToOverrides(
persisted.agents?.providers as Record<string, ProviderOverride> | undefined,
@@ -208,6 +209,10 @@ function mergeMutableConfigIntoPersistedConfig(params: {
...persisted.daemon?.mcp,
injectIntoAgents: mutable.mcp.injectIntoAgents,
},
browserTools: {
...persisted.daemon?.browserTools,
enabled: browserToolsEnabled,
},
autoArchiveAfterMerge: mutable.autoArchiveAfterMerge,
enableTerminalAgentHooks: mutable.enableTerminalAgentHooks,
appendSystemPrompt: mutable.appendSystemPrompt,
@@ -219,6 +224,14 @@ function mergeMutableConfigIntoPersistedConfig(params: {
} as PersistedConfig;
}
function readBrowserToolsEnabled(mutable: MutableDaemonConfig): boolean {
const browserTools = mutable.browserTools;
if (!isRecord(browserTools)) {
return false;
}
return browserTools["enabled"] === true;
}
function readMetadataGenerationProviders(
mutable: MutableDaemonConfig,
): Array<{ provider: string; model?: string; thinkingOptionId?: string }> {
@@ -46,6 +46,18 @@ describe("PersistedConfigSchema daemon append system prompt config", () => {
});
});
describe("PersistedConfigSchema daemon browser tools config", () => {
test("accepts optional browser tools opt-in", () => {
const parsed = PersistedConfigSchema.parse({
daemon: {
browserTools: { enabled: true },
},
});
expect(parsed.daemon?.browserTools?.enabled).toBe(true);
});
});
describe("PersistedConfigSchema daemon relay config", () => {
test("accepts optional relay TLS setting", () => {
const parsed = PersistedConfigSchema.parse({
@@ -240,6 +240,12 @@ export const PersistedConfigSchema = z
})
.passthrough()
.optional(),
browserTools: z
.object({
enabled: z.boolean().optional(),
})
.passthrough()
.optional(),
autoArchiveAfterMerge: z.boolean().optional(),
enableTerminalAgentHooks: z.boolean().optional(),
appendSystemPrompt: z.string().optional(),
@@ -3816,7 +3816,7 @@ test("open_project_request does not match a new child directory to an existing p
const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const home = path.resolve("/Users/moboudra");
const home = path.resolve("/home/developer");
const worktree = path.join(home, ".paseo", "worktrees", "project-config-lifecycle-textarea");
projects.set(
@@ -3825,7 +3825,7 @@ test("open_project_request does not match a new child directory to an existing p
projectId: home,
rootPath: home,
kind: "non_git",
displayName: "moboudra",
displayName: "developer",
createdAt: "2026-04-24T09:00:00.000Z",
updatedAt: "2026-04-24T09:00:00.000Z",
}),
@@ -3837,7 +3837,7 @@ test("open_project_request does not match a new child directory to an existing p
projectId: home,
cwd: home,
kind: "directory",
displayName: "moboudra",
displayName: "developer",
createdAt: "2026-04-24T09:00:00.000Z",
updatedAt: "2026-04-24T09:00:00.000Z",
}),
@@ -3882,7 +3882,7 @@ test("open_project_request does not unarchive an archived parent workspace for a
const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const home = path.resolve("/Users/moboudra");
const home = path.resolve("/home/developer");
const worktree = path.join(home, ".paseo", "worktrees", "project-config-lifecycle-textarea");
const archivedAt = "2026-04-24T08:00:00.000Z";
@@ -3950,9 +3950,9 @@ test("open_project_request reclassifies an archived directory workspace when git
const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const repoRoot = path.resolve("/Users/moboudra/dev/paseo");
const repoRoot = path.resolve("/home/developer/dev/paseo");
const cwd = path.join(
path.resolve("/Users/moboudra"),
path.resolve("/home/developer"),
".paseo",
"worktrees",
"orchestrate",
@@ -4049,9 +4049,9 @@ test("open_project_request reclassifies an active directory workspace when git m
const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const repoRoot = path.resolve("/Users/moboudra/dev/paseo");
const repoRoot = path.resolve("/home/developer/dev/paseo");
const cwd = path.join(
path.resolve("/Users/moboudra"),
path.resolve("/home/developer"),
".paseo",
"worktrees",
"orchestrate",
@@ -4167,9 +4167,9 @@ test("open_project_request groups a plain git worktree under an existing repo pr
const session = createSessionForWorkspaceTests();
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>();
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>();
const repoRoot = path.resolve("/Users/moboudra/dev/paseo");
const repoRoot = path.resolve("/home/developer/dev/paseo");
const cwd = path.join(
path.resolve("/Users/moboudra"),
path.resolve("/home/developer"),
".paseo",
"worktrees",
"orchestrate",
@@ -12,7 +12,7 @@ import {
export type DaemonClientConfig = Omit<
SharedDaemonClientConfig,
"webSocketFactory" | "transportFactory" | "clientId"
>;
> & { clientId?: string };
export type CreateAgentOptions = CreateAgentRequestOptions;
export { type SendMessageOptions, type DaemonEvent, type DaemonEventHandler };
@@ -25,7 +25,7 @@ function nextTestClientId(): string {
export class DaemonClient extends SharedDaemonClient {
constructor(config: DaemonClientConfig) {
const clientId = nextTestClientId();
const clientId = config.clientId ?? nextTestClientId();
super({
...config,
clientId,
@@ -0,0 +1,406 @@
import { createServer, type Server as HTTPServer } from "node:http";
import type { AddressInfo } from "node:net";
import type {
BrowserAutomationExecuteRequest,
BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
import type pino from "pino";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { AgentManager } from "./agent/agent-manager.js";
import type { AgentStorage } from "./agent/agent-storage.js";
import { BrowserToolsBroker } from "./browser-tools/broker.js";
import { StaticBrowserToolsPolicy } from "./browser-tools/policy.js";
import type { CheckoutDiffManager } from "./checkout-diff-manager.js";
import type { FileBackedChatService } from "./chat/chat-service.js";
import type { DaemonConfigStore } from "./daemon-config-store.js";
import type { DownloadTokenStore } from "./file-download/token-store.js";
import type { LoopService } from "./loop-service.js";
import type { ScheduleService } from "./schedule/service.js";
import { createStub } from "./test-utils/class-mocks.js";
import { DaemonClient } from "./test-utils/daemon-client.js";
import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js";
import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
interface BrowserToolsDaemonHarness {
broker: BrowserToolsBroker;
connectDesktopBrowserClient(
options?: ConnectDesktopBrowserClientOptions,
): Promise<DesktopBrowserClientHandle>;
stop(): Promise<void>;
}
interface ConnectDesktopBrowserClientOptions {
clientId?: string;
capabilities?: Partial<Record<string, boolean>>;
}
interface DesktopBrowserClientHandle {
clientId: string;
nextBrowserRequest(): Promise<BrowserAutomationExecuteRequest>;
respondToBrowserRequest(response: BrowserAutomationExecuteResponse): void;
disconnect(): Promise<void>;
}
interface QueuedBrowserRequests {
next(): Promise<BrowserAutomationExecuteRequest>;
push(request: BrowserAutomationExecuteRequest): void;
close(): void;
}
const harnesses: BrowserToolsDaemonHarness[] = [];
afterEach(async () => {
await Promise.all(harnesses.splice(0).map((harness) => harness.stop()));
});
describe("WebSocketServer browser tools wiring", () => {
it("registers capable clients and dispatches broker requests over the real WebSocket path", async () => {
const harness = await startBrowserToolsDaemonHarness();
const desktop = await harness.connectDesktopBrowserClient();
const resultPromise = harness.broker.execute({
command: { command: "list_tabs", args: {} },
});
const request = await desktop.nextBrowserRequest();
expect(request).toMatchObject({
type: "browser.automation.execute.request",
requestId: "req-1",
command: { command: "list_tabs", args: {} },
});
desktop.respondToBrowserRequest({
type: "browser.automation.execute.response",
payload: {
requestId: request.requestId,
ok: true,
result: { command: "list_tabs", tabs: [] },
},
});
await expect(resultPromise).resolves.toEqual({
requestId: request.requestId,
ok: true,
result: { command: "list_tabs", tabs: [] },
});
});
it("unregisters capable clients on disconnect and clears pending browser commands", async () => {
const harness = await startBrowserToolsDaemonHarness();
const desktop = await harness.connectDesktopBrowserClient();
const pendingResult = harness.broker.execute({
command: { command: "list_tabs", args: {} },
});
const pendingExpectation = expect(pendingResult).resolves.toMatchObject({
ok: false,
error: { code: "browser_no_desktop", retryable: true },
});
await desktop.nextBrowserRequest();
expect(harness.broker.getPendingRequestCount()).toBe(1);
await desktop.disconnect();
expect(harness.broker.getRegisteredClientCount()).toBe(0);
expect(harness.broker.getPendingRequestCount()).toBe(0);
await pendingExpectation;
await expect(
harness.broker.execute({ command: { command: "list_tabs", args: {} } }),
).resolves.toMatchObject({
ok: false,
error: { code: "browser_no_desktop" },
});
});
it("keeps browser automation registered when a desktop browser client resumes", async () => {
const harness = await startBrowserToolsDaemonHarness();
const clientId = "desktop-client-1";
await harness.connectDesktopBrowserClient({
clientId,
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
});
const resumedDesktop = await harness.connectDesktopBrowserClient({
clientId,
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
});
const resultPromise = harness.broker.execute({
command: { command: "click", args: { ref: "@e1" } },
});
const request = await resumedDesktop.nextBrowserRequest();
resumedDesktop.respondToBrowserRequest({
type: "browser.automation.execute.response",
payload: {
requestId: request.requestId,
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e1" },
},
});
await expect(resultPromise).resolves.toMatchObject({
ok: true,
result: { command: "click", browserId: "browser-1", ref: "@e1" },
});
});
});
async function startBrowserToolsDaemonHarness(): Promise<BrowserToolsDaemonHarness> {
const httpServer = createServer();
const broker = createBroker();
const wsServer = createVoiceAssistantWebSocketServer({ httpServer, broker });
const clients = new Set<DaemonClient>();
await listen(httpServer);
const url = `ws://127.0.0.1:${getPort(httpServer)}/ws`;
const harness: BrowserToolsDaemonHarness = {
broker,
async connectDesktopBrowserClient(options = {}) {
const clientId = options.clientId;
const client = new DaemonClient({
url,
...(clientId ? { clientId } : {}),
clientType: "browser",
connectTimeoutMs: 500,
reconnect: { enabled: false },
capabilities: options.capabilities ?? { [CLIENT_CAPS.desktopBrowserAutomation]: true },
});
clients.add(client);
const requests = createBrowserRequestQueue();
client.on("browser.automation.execute.request", (request) => {
requests.push(request);
});
await client.connect();
return {
clientId: clientId ?? "",
nextBrowserRequest: () => requests.next(),
respondToBrowserRequest: (response) =>
client.sendBrowserAutomationExecuteResponse(response),
async disconnect() {
requests.close();
clients.delete(client);
await client.close();
await waitFor(() => broker.getRegisteredClientCount() === 0);
},
};
},
async stop() {
await Promise.all(Array.from(clients, (client) => client.close()));
clients.clear();
await wsServer.close();
await closeHttpServer(httpServer);
},
};
harnesses.push(harness);
return harness;
}
function createBroker(): BrowserToolsBroker {
return new BrowserToolsBroker({
policy: new StaticBrowserToolsPolicy(true),
defaultTimeoutMs: 500,
createRequestId: createRequestIdSequence(),
});
}
function createRequestIdSequence(): () => string {
let index = 0;
return () => {
index += 1;
return `req-${index}`;
};
}
function createVoiceAssistantWebSocketServer(params: {
httpServer: HTTPServer;
broker: BrowserToolsBroker;
}): VoiceAssistantWebSocketServer {
const { httpServer, broker } = params;
const agentManager = {
setAgentAttentionCallback: vi.fn(),
subscribe: vi.fn(() => () => {}),
getMetricsSnapshot: vi.fn(() => ({
total: 0,
byLifecycle: {},
withActiveForegroundTurn: 0,
timelineStats: { totalItems: 0, maxItemsPerAgent: 0 },
})),
};
const daemonConfigStore = {
onChange: vi.fn(() => () => {}),
};
return new VoiceAssistantWebSocketServer(
httpServer,
createStub<pino.Logger>(createLogger()),
"srv-test",
createStub<AgentManager>(agentManager),
createStub<AgentStorage>({}),
createStub<DownloadTokenStore>({}),
"/tmp/paseo-browser-tools-websocket-test",
createStub<DaemonConfigStore>(daemonConfigStore),
null,
{ allowedOrigins: new Set(["*"]) },
undefined,
undefined,
undefined,
undefined,
"1.2.3-test",
undefined,
undefined,
undefined,
createStub<FileBackedChatService>({}),
createStub<LoopService>({}),
createStub<ScheduleService>({}),
createStub<CheckoutDiffManager>({
subscribe: vi.fn(),
scheduleRefreshForCwd: vi.fn(),
getMetrics: vi.fn(() => ({
checkoutDiffTargetCount: 0,
checkoutDiffSubscriptionCount: 0,
checkoutDiffWatcherCount: 0,
checkoutDiffFallbackRefreshTargetCount: 0,
})),
dispose: vi.fn(),
}),
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
createProviderSnapshotManagerStub().manager,
undefined,
undefined,
broker,
);
}
function createBrowserRequestQueue(): QueuedBrowserRequests {
const requests: BrowserAutomationExecuteRequest[] = [];
const waiters: Array<{
resolve: (request: BrowserAutomationExecuteRequest) => void;
reject: (error: Error) => void;
}> = [];
let closed = false;
return {
next() {
const request = requests.shift();
if (request) {
return Promise.resolve(request);
}
if (closed) {
return Promise.reject(new Error("Desktop browser client disconnected"));
}
return new Promise<BrowserAutomationExecuteRequest>((resolve, reject) => {
let timeout: ReturnType<typeof setTimeout>;
const waiter = {
resolve: (value: BrowserAutomationExecuteRequest) => {
clearTimeout(timeout);
resolve(value);
},
reject: (error: Error) => {
clearTimeout(timeout);
reject(error);
},
};
timeout = setTimeout(() => {
const waiterIndex = waiters.indexOf(waiter);
if (waiterIndex !== -1) {
waiters.splice(waiterIndex, 1);
}
reject(new Error("Timed out waiting for browser automation request"));
}, 500);
waiters.push(waiter);
});
},
push(request) {
const waiter = waiters.shift();
if (waiter) {
waiter.resolve(request);
return;
}
requests.push(request);
},
close() {
closed = true;
for (const waiter of waiters.splice(0)) {
waiter.reject(new Error("Desktop browser client disconnected"));
}
},
};
}
function createLogger() {
const logger = {
child: vi.fn(() => logger),
trace: vi.fn(),
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
};
return logger;
}
function listen(server: HTTPServer): Promise<void> {
return new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", () => {
server.off("error", reject);
resolve();
});
});
}
function getPort(server: HTTPServer): number {
const address = server.address();
if (!isAddressInfo(address)) {
throw new Error("HTTP test server did not bind to a TCP port");
}
return address.port;
}
function isAddressInfo(address: string | AddressInfo | null): address is AddressInfo {
return typeof address === "object" && address !== null && typeof address.port === "number";
}
function closeHttpServer(server: HTTPServer): Promise<void> {
if (!server.listening) {
return Promise.resolve();
}
return new Promise((resolve, reject) => {
server.close((error) => {
if (error) {
reject(error);
return;
}
resolve();
});
});
}
async function waitFor(predicate: () => boolean): Promise<void> {
const startedAt = Date.now();
while (!predicate()) {
if (Date.now() - startedAt > 500) {
throw new Error("Timed out waiting for browser tools WebSocket state");
}
await new Promise((resolve) => setTimeout(resolve, 10));
}
}
@@ -69,6 +69,9 @@ import {
CLIENT_SHUTDOWN_RPC_REASON,
normalizeClientRestartRpcReason,
} from "./lifecycle-reasons.js";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
import type { BrowserAutomationExecuteResponse } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { BrowserToolsBroker } from "./browser-tools/broker.js";
const WS_CLOSE_DAEMON_AUTH_FAILED = 4401;
@@ -304,6 +307,12 @@ function bufferFromWsData(data: Buffer | ArrayBuffer | Buffer[] | string): Buffe
return Buffer.from(data);
}
function hasDesktopBrowserAutomationCapability(
capabilities: Record<string, unknown> | null,
): boolean {
return capabilities?.[CLIENT_CAPS.desktopBrowserAutomation] === true;
}
interface WebSocketLike {
readyState: number;
bufferedAmount?: number;
@@ -323,6 +332,10 @@ interface SessionConnection {
externalDisconnectCleanupTimeout: ReturnType<typeof setTimeout> | null;
}
interface BrowserToolsRegistration {
unregister: () => void;
}
const SLOW_REQUEST_THRESHOLD_MS = 500;
const EXTERNAL_SESSION_DISCONNECT_GRACE_MS = 90_000;
const HELLO_TIMEOUT_MS = 15_000;
@@ -438,6 +451,8 @@ export class VoiceAssistantWebSocketServer {
private unsubscribeDaemonConfigChange: (() => void) | null = null;
private readonly providerUsageService: ProviderUsageService;
private unsubscribeTerminalActivity: (() => void) | null = null;
private readonly browserToolsBroker: BrowserToolsBroker | null;
private readonly browserToolsRegistrations = new Map<string, BrowserToolsRegistration>();
constructor(
server: HTTPServer,
@@ -491,6 +506,7 @@ export class VoiceAssistantWebSocketServer {
};
},
serviceProxyPublicBaseUrl?: string | null,
browserToolsBroker?: BrowserToolsBroker | null,
) {
this.logger = logger.child({ module: "websocket-server" });
this.serverId = serverId;
@@ -499,6 +515,7 @@ export class VoiceAssistantWebSocketServer {
}
this.daemonVersion = daemonVersion.trim();
this.daemonRuntimeConfig = daemonRuntimeConfig;
this.browserToolsBroker = browserToolsBroker ?? null;
this.agentManager = agentManager;
this.agentStorage = agentStorage;
this.projectRegistry = projectRegistry ?? createNoopProjectRegistry();
@@ -842,6 +859,9 @@ export class VoiceAssistantWebSocketServer {
this.sessions.clear();
this.socketIdentities.clear();
this.externalSessionsByKey.clear();
for (const clientId of this.browserToolsRegistrations.keys()) {
this.unregisterBrowserToolsClient(clientId);
}
this.wss.close();
}
@@ -1124,10 +1144,12 @@ export class VoiceAssistantWebSocketServer {
) {
existing.clientCapabilities = newClientCapabilities;
existing.session.updateClientCapabilities(newClientCapabilities);
this.syncBrowserToolsClientRegistration(existing);
}
existing.sockets.add(ws);
this.sessions.set(ws, existing);
pending.identity.sessionId = existing.session.getSessionId();
this.syncBrowserToolsClientRegistration(existing);
this.sendToClient(ws, this.createServerInfoMessage());
pending.connectionLogger.info(
{
@@ -1152,6 +1174,7 @@ export class VoiceAssistantWebSocketServer {
this.sessions.set(ws, connection);
this.externalSessionsByKey.set(clientId, connection);
pending.identity.sessionId = connection.session.getSessionId();
this.syncBrowserToolsClientRegistration(connection);
this.sendToClient(ws, this.createServerInfoMessage());
connection.connectionLogger.info(
{
@@ -1314,6 +1337,7 @@ export class VoiceAssistantWebSocketServer {
this.socketIdentities.delete(ws);
if (connection.sockets.size === 0) {
this.unregisterBrowserToolsClient(connection.clientId);
this.incrementRuntimeCounter("sessionDisconnectedWaitingReconnect");
if (connection.externalDisconnectCleanupTimeout) {
clearTimeout(connection.externalDisconnectCleanupTimeout);
@@ -1375,6 +1399,7 @@ export class VoiceAssistantWebSocketServer {
if (existing === connection) {
this.externalSessionsByKey.delete(connection.clientId);
}
this.unregisterBrowserToolsClient(connection.clientId);
connection.connectionLogger.trace(
{ clientId: connection.clientId, totalSessions: this.sessions.size },
@@ -1383,6 +1408,39 @@ export class VoiceAssistantWebSocketServer {
await connection.session.cleanup();
}
private syncBrowserToolsClientRegistration(connection: SessionConnection): void {
if (!this.browserToolsBroker) {
return;
}
if (!hasDesktopBrowserAutomationCapability(connection.clientCapabilities)) {
this.unregisterBrowserToolsClient(connection.clientId);
return;
}
const existing = this.browserToolsRegistrations.get(connection.clientId);
if (existing) {
return;
}
const unregister = this.browserToolsBroker.registerClient({
id: connection.clientId,
sendBrowserAutomationRequest: (request) => {
this.sendToConnection(connection, wrapSessionMessage(request));
},
});
this.browserToolsRegistrations.set(connection.clientId, {
unregister,
});
}
private unregisterBrowserToolsClient(clientId: string): void {
const registration = this.browserToolsRegistrations.get(clientId);
if (!registration) {
return;
}
this.browserToolsRegistrations.delete(clientId);
registration.unregister();
}
private handleInvalidInboundMessage(args: {
ws: WebSocketLike;
parsed: unknown;
@@ -1625,6 +1683,11 @@ export class VoiceAssistantWebSocketServer {
"ws_control_rpc_received",
);
}
if (message.message.type === "browser.automation.execute.response") {
this.browserToolsBroker?.receiveResponse(message.message as BrowserAutomationExecuteResponse);
return;
}
const startMs = performance.now();
await activeConnection.session.handleMessage(message.message);
const durationMs = performance.now() - startMs;