diff --git a/packages/app/src/browser-automation/handler.test.ts b/packages/app/src/browser-automation/handler.test.ts index 45ca127c8..756c7262b 100644 --- a/packages/app/src/browser-automation/handler.test.ts +++ b/packages/app/src/browser-automation/handler.test.ts @@ -396,7 +396,7 @@ describe("mountBrowserAutomationHandler", () => { ok: false, error: { code: "browser_unsupported", - message: "Desktop browser automation is not available in this app runtime.", + message: "Browser automation is not available in this app runtime.", retryable: false, }, }, @@ -450,7 +450,7 @@ describe("mountBrowserAutomationHandler", () => { ok: false, error: { code: "browser_unsupported", - message: "Desktop browser automation is not implemented by this desktop build yet.", + message: "Browser automation is not implemented by this app build yet.", retryable: false, }, }, diff --git a/packages/app/src/browser-automation/handler.ts b/packages/app/src/browser-automation/handler.ts index 5647148a7..44677ac0a 100644 --- a/packages/app/src/browser-automation/handler.ts +++ b/packages/app/src/browser-automation/handler.ts @@ -120,7 +120,7 @@ async function handleBrowserAutomationRequest(params: { payload: browserAutomationFailure({ requestId: request.requestId, code: "browser_unsupported", - message: "Desktop browser automation is not available in this app runtime.", + message: "Browser automation is not available in this app runtime.", }), }); return; @@ -202,7 +202,7 @@ async function openBrowserTabForRequest(params: { return browserAutomationFailure({ requestId: request.requestId, code: "browser_timeout", - message: `Timed out waiting for browser tab ${browserId} to register with desktop automation. Try browser_new_tab again.`, + message: `Timed out waiting for browser tab ${browserId} to register with the browser automation host. Try browser_new_tab again.`, retryable: true, }); } @@ -270,14 +270,14 @@ function normalizeThrownBridgeError( return browserAutomationFailure({ requestId, code: "browser_unsupported", - message: "Desktop browser automation is not implemented by this desktop build yet.", + message: "Browser automation is not implemented by this app build yet.", }); } return browserAutomationFailure({ requestId, code: "browser_unknown_error", - message: message || "Desktop browser automation failed.", + message: message || "Browser automation failed.", }); } diff --git a/packages/app/src/runtime/host-runtime.ts b/packages/app/src/runtime/host-runtime.ts index 23633c3b2..8cee8a3b0 100644 --- a/packages/app/src/runtime/host-runtime.ts +++ b/packages/app/src/runtime/host-runtime.ts @@ -39,6 +39,7 @@ import { } from "@/desktop/daemon/desktop-daemon-transport"; import { getDesktopHost } from "@/desktop/host"; import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; +import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { replaceFetchedAgentDirectory } from "@/utils/agent-directory-sync"; import { useSessionStore } from "@/stores/session-store"; import { @@ -522,10 +523,15 @@ function probeIntervalForConnection( } function createDefaultDeps(): HostRuntimeControllerDeps { - const desktopBrowserAutomationAvailable = + const browserHostAvailable = typeof getDesktopHost()?.browser?.executeAutomationCommand === "function"; - const browserAutomationCapabilities = desktopBrowserAutomationAvailable - ? { [CLIENT_CAPS.desktopBrowserAutomation]: true } + const browserAutomationCapabilities = browserHostAvailable + ? { + [CLIENT_CAPS.browserHost]: { + supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], + hostKind: "desktop app", + }, + } : undefined; return { diff --git a/packages/app/src/screens/settings/browser-tools-config.ts b/packages/app/src/screens/settings/browser-tools-config.ts index c84b92ec3..ab009eb6d 100644 --- a/packages/app/src/screens/settings/browser-tools-config.ts +++ b/packages/app/src/screens/settings/browser-tools-config.ts @@ -2,7 +2,7 @@ import type { MutableDaemonConfig } from "@getpaseo/protocol/messages"; export const BROWSER_TOOLS_TITLE = "Browser tools"; export const BROWSER_TOOLS_WARNING = - "Allow agents to access and control Paseo desktop browser tabs, including logged-in browser state. Only enable this for agents you trust."; + "Allow agents to access and control Paseo browser tabs, including logged-in browser state. Only enable this for agents you trust."; export interface BrowserToolsCardState { isVisible: boolean; diff --git a/packages/client/src/daemon-client.test.ts b/packages/client/src/daemon-client.test.ts index ce927e0d8..910fdd170 100644 --- a/packages/client/src/daemon-client.test.ts +++ b/packages/client/src/daemon-client.test.ts @@ -2,6 +2,7 @@ import { afterEach, expect, expectTypeOf, test, vi } from "vitest"; import { z } from "zod"; import { DaemonClient, type DaemonTransport, type Logger } from "./daemon-client"; import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; +import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { decodeFileTransferFrame, encodeFileTransferFrame, @@ -164,7 +165,7 @@ test("does not infer browser automation capabilities from Electron runtime", asy capabilities: z.record(z.unknown()), }) .parse(JSON.parse(assertStr(mock.sent[0]))); - expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBeUndefined(); + expect(hello.capabilities[CLIENT_CAPS.browserHost]).toBeUndefined(); }); test("advertises consumer-provided browser automation capabilities", async () => { @@ -174,7 +175,12 @@ test("advertises consumer-provided browser automation capabilities", async () => clientId: "browser_capability_unit_test", transportFactory: () => mock.transport, reconnect: { enabled: false }, - capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true }, + capabilities: { + [CLIENT_CAPS.browserHost]: { + supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], + hostKind: "desktop app", + }, + }, }); clients.push(client); @@ -188,7 +194,10 @@ test("advertises consumer-provided browser automation capabilities", async () => capabilities: z.record(z.unknown()), }) .parse(JSON.parse(assertStr(mock.sent[0]))); - expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBe(true); + expect(hello.capabilities[CLIENT_CAPS.browserHost]).toEqual({ + supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], + hostKind: "desktop app", + }); }); const noopLogger: Logger = { @@ -506,7 +515,12 @@ test("advertises client capabilities in hello", async () => { logger, reconnect: { enabled: false }, transportFactory: () => mock.transport, - capabilities: { desktop_browser_automation: true }, + capabilities: { + browser_host: { + supportedCommands: ["list_tabs"], + hostKind: "desktop app", + }, + }, }); clients.push(client); @@ -524,7 +538,10 @@ test("advertises client capabilities in hello", async () => { custom_mode_icons: true, reasoning_merge_enum: true, terminal_reflowable_snapshot: true, - desktop_browser_automation: true, + browser_host: { + supportedCommands: ["list_tabs"], + hostKind: "desktop app", + }, }, }); }); diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts index 4218cbbce..b85e2acac 100644 --- a/packages/client/src/daemon-client.ts +++ b/packages/client/src/daemon-client.ts @@ -252,7 +252,7 @@ export interface DaemonClientConfig { }; runtimeMetricsIntervalMs?: number; runtimeMetricsWindowMs?: number; - capabilities?: Partial>; + capabilities?: Partial>; } export interface SendMessageOptions { diff --git a/packages/protocol/src/browser-automation/capabilities.ts b/packages/protocol/src/browser-automation/capabilities.ts new file mode 100644 index 000000000..415c6b442 --- /dev/null +++ b/packages/protocol/src/browser-automation/capabilities.ts @@ -0,0 +1,41 @@ +import { z } from "zod"; +import { + BROWSER_AUTOMATION_COMMAND_NAMES, + type BrowserAutomationCommandName, +} from "./rpc-schemas.js"; + +const KNOWN_BROWSER_AUTOMATION_COMMAND_NAMES = new Set(BROWSER_AUTOMATION_COMMAND_NAMES); + +export const BrowserAutomationHostCapabilitySchema = z + .object({ + supportedCommands: z.array(z.string().min(1)).transform((commands, context) => { + const supportedCommands: BrowserAutomationCommandName[] = []; + const seen = new Set(); + + for (const command of commands) { + if (!isKnownBrowserAutomationCommandName(command) || seen.has(command)) { + continue; + } + seen.add(command); + supportedCommands.push(command); + } + + if (supportedCommands.length === 0) { + context.addIssue({ + code: "custom", + message: "supportedCommands must include at least one known browser automation command", + }); + return z.NEVER; + } + + return supportedCommands; + }), + hostKind: z.string().min(1).default("browser host"), + }) + .passthrough(); + +export type BrowserAutomationHostCapability = z.infer; + +function isKnownBrowserAutomationCommandName(value: string): value is BrowserAutomationCommandName { + return KNOWN_BROWSER_AUTOMATION_COMMAND_NAMES.has(value); +} diff --git a/packages/protocol/src/browser-automation/rpc-schemas.test.ts b/packages/protocol/src/browser-automation/rpc-schemas.test.ts index 3bf53e19e..39e9b2ac8 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.test.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.test.ts @@ -564,8 +564,8 @@ describe("browser automation execute RPC schemas", () => { requestId: "req-error", ok: false, error: { - code: "browser_no_desktop", - message: "No desktop browser automation client is connected.", + code: "browser_no_host", + message: "No browser automation host is connected.", }, }, }); @@ -574,8 +574,8 @@ describe("browser automation execute RPC schemas", () => { requestId: "req-error", ok: false, error: { - code: "browser_no_desktop", - message: "No desktop browser automation client is connected.", + code: "browser_no_host", + message: "No browser automation host is connected.", retryable: false, }, }); diff --git a/packages/protocol/src/browser-automation/rpc-schemas.ts b/packages/protocol/src/browser-automation/rpc-schemas.ts index 593b15c13..d3cdedd49 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.ts @@ -2,7 +2,7 @@ import { z } from "zod"; export const BrowserAutomationErrorCodeSchema = z.enum([ "browser_disabled", - "browser_no_desktop", + "browser_no_host", "browser_tab_not_found", "browser_tab_closed", "browser_timeout", @@ -20,6 +20,29 @@ const BROWSER_AUTOMATION_BROWSER_ID_MESSAGE = const BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE = "browser_wait requires exactly one of text or url"; +export const BROWSER_AUTOMATION_COMMAND_NAMES = [ + "list_tabs", + "new_tab", + "snapshot", + "click", + "fill", + "wait", + "type", + "keypress", + "navigate", + "back", + "forward", + "reload", + "screenshot", + "upload", + "select", + "hover", + "drag", + "logs", +] as const; + +export const BrowserAutomationCommandNameSchema = z.enum(BROWSER_AUTOMATION_COMMAND_NAMES); + export const BrowserAutomationBrowserIdSchema = z .string({ error: () => BROWSER_AUTOMATION_BROWSER_ID_MESSAGE }) .min(1, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE) @@ -399,6 +422,7 @@ export const BrowserAutomationExecuteResponseSchema = z.object({ }); export type BrowserAutomationErrorCode = z.infer; +export type BrowserAutomationCommandName = z.infer; export type BrowserAutomationCommand = z.infer; export type BrowserAutomationResult = z.infer; export type BrowserAutomationConsoleLogEntry = z.infer< diff --git a/packages/protocol/src/client-capabilities.ts b/packages/protocol/src/client-capabilities.ts index 3ec353439..1b315d5d9 100644 --- a/packages/protocol/src/client-capabilities.ts +++ b/packages/protocol/src/client-capabilities.ts @@ -11,7 +11,7 @@ export const CLIENT_CAPS = { // Old clients use a strict TerminalState schema and would reject the extra fields. // Drop the gate (always send the flags) when floor >= v0.1.88. terminalReflowableSnapshot: "terminal_reflowable_snapshot", - desktopBrowserAutomation: "desktop_browser_automation", + browserHost: "browser_host", } as const; export type ClientCapability = (typeof CLIENT_CAPS)[keyof typeof CLIENT_CAPS]; diff --git a/packages/protocol/src/messages.browser-automation.test.ts b/packages/protocol/src/messages.browser-automation.test.ts index 5b3a8a721..7fb255dd1 100644 --- a/packages/protocol/src/messages.browser-automation.test.ts +++ b/packages/protocol/src/messages.browser-automation.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "vitest"; +import { BROWSER_AUTOMATION_COMMAND_NAMES } from "./browser-automation/rpc-schemas.js"; import { CLIENT_CAPS } from "./client-capabilities.js"; import { MutableDaemonConfigPatchSchema, @@ -12,7 +13,7 @@ import { describe("browser automation protocol integration", () => { const browserId = "11111111-1111-4111-8111-111111111111"; - test("desktop automation capability parses in hello without narrowing old clients", () => { + test("browser host capability parses supported commands in hello", () => { expect( WSHelloMessageSchema.parse({ type: "hello", @@ -20,13 +21,86 @@ describe("browser automation protocol integration", () => { clientType: "mobile", protocolVersion: 1, capabilities: { - [CLIENT_CAPS.desktopBrowserAutomation]: true, + [CLIENT_CAPS.browserHost]: { + supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], + hostKind: "desktop app", + }, }, }).capabilities, ).toMatchObject({ - [CLIENT_CAPS.desktopBrowserAutomation]: true, + [CLIENT_CAPS.browserHost]: { + supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], + hostKind: "desktop app", + }, }); + }); + test("browser host capability requires at least one supported command", () => { + expect(() => + WSHelloMessageSchema.parse({ + type: "hello", + clientId: "client-1", + clientType: "mobile", + protocolVersion: 1, + capabilities: { + [CLIENT_CAPS.browserHost]: { + supportedCommands: [], + hostKind: "desktop app", + }, + }, + }), + ).toThrow(); + + expect(() => + WSHelloMessageSchema.parse({ + type: "hello", + clientId: "client-2", + clientType: "mobile", + protocolVersion: 1, + capabilities: { + [CLIENT_CAPS.browserHost]: {}, + }, + }), + ).toThrow(); + + expect(() => + WSHelloMessageSchema.parse({ + type: "hello", + clientId: "client-3", + clientType: "mobile", + protocolVersion: 1, + capabilities: { + [CLIENT_CAPS.browserHost]: { + supportedCommands: ["future_command"], + }, + }, + }), + ).toThrow(); + }); + + test("browser host capability ignores unknown future commands when known commands remain", () => { + expect( + WSHelloMessageSchema.parse({ + type: "hello", + clientId: "client-1", + clientType: "mobile", + protocolVersion: 1, + capabilities: { + [CLIENT_CAPS.browserHost]: { + supportedCommands: ["list_tabs", "future_command", "list_tabs"], + hostKind: "desktop app", + }, + }, + }).capabilities, + ).toMatchObject({ + [CLIENT_CAPS.browserHost]: { + supportedCommands: ["list_tabs"], + hostKind: "desktop app", + }, + }); + }); + + test("hello remains valid when no browser host capability is advertised", () => { expect( WSHelloMessageSchema.parse({ type: "hello", @@ -37,7 +111,7 @@ describe("browser automation protocol integration", () => { ).toBeUndefined(); }); - test("daemon to desktop execute request is an outbound session message", () => { + test("daemon to browser host execute request is an outbound session message", () => { const parsed = SessionOutboundMessageSchema.parse({ type: "browser.automation.execute.request", requestId: "req-1", @@ -47,7 +121,7 @@ describe("browser automation protocol integration", () => { expect(parsed.type).toBe("browser.automation.execute.request"); }); - test("desktop to daemon execute response is an inbound session message", () => { + test("browser host to daemon execute response is an inbound session message", () => { const parsed = SessionInboundMessageSchema.parse({ type: "browser.automation.execute.response", payload: { diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 8e726078e..b91aa0d82 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -57,6 +57,7 @@ import { BrowserAutomationExecuteRequestSchema, BrowserAutomationExecuteResponseSchema, } from "./browser-automation/rpc-schemas.js"; +import { BrowserAutomationHostCapabilitySchema } from "./browser-automation/capabilities.js"; import { PaseoConfigRawSchema, PaseoLifecycleCommandRawSchema, @@ -4672,7 +4673,7 @@ export const WSHelloMessageSchema = z.object({ [CLIENT_CAPS.reasoningMergeEnum]: z.boolean().optional(), [CLIENT_CAPS.customModeIcons]: z.boolean().optional(), [CLIENT_CAPS.terminalReflowableSnapshot]: z.boolean().optional(), - [CLIENT_CAPS.desktopBrowserAutomation]: z.boolean().optional(), + [CLIENT_CAPS.browserHost]: BrowserAutomationHostCapabilitySchema.optional(), }) .passthrough() .optional(), diff --git a/packages/server/src/server/browser-tools/broker.test.ts b/packages/server/src/server/browser-tools/broker.test.ts index 2c92f6581..d73f63e84 100644 --- a/packages/server/src/server/browser-tools/broker.test.ts +++ b/packages/server/src/server/browser-tools/broker.test.ts @@ -1,18 +1,32 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import type { BrowserAutomationCommand, + BrowserAutomationCommandName, BrowserAutomationExecuteRequest, BrowserAutomationExecuteResponse, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; -import { BrowserToolsBroker, type BrowserToolsDesktopClient } from "./broker.js"; +import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import { BrowserToolsBroker, type BrowserHostClient } from "./broker.js"; import { StaticBrowserToolsPolicy } from "./policy.js"; const BROWSER_ID = "11111111-1111-4111-8111-111111111111"; +const SECOND_BROWSER_ID = "22222222-2222-4222-8222-222222222222"; -class FakeDesktopClient implements BrowserToolsDesktopClient { +class FakeBrowserHostClient implements BrowserHostClient { public readonly receivedRequests: BrowserAutomationExecuteRequest[] = []; + public readonly hostKind: string; + public readonly supportedCommands: readonly BrowserAutomationCommandName[]; - public constructor(public readonly id: string) {} + public constructor( + public readonly id: string, + options: { + hostKind?: string; + supportedCommands?: readonly BrowserAutomationCommandName[]; + } = {}, + ) { + this.hostKind = options.hostKind ?? "desktop app"; + this.supportedCommands = options.supportedCommands ?? [...BROWSER_AUTOMATION_COMMAND_NAMES]; + } public sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void { this.receivedRequests.push(request); @@ -21,16 +35,30 @@ class FakeDesktopClient implements BrowserToolsDesktopClient { public resolveLatestWith( broker: BrowserToolsBroker, responsePayload: BrowserAutomationExecuteResponse["payload"], + ): boolean { + const latest = this.receivedRequests.at(-1); + if (!latest) { + throw new Error(`Host ${this.id} has not received a browser request.`); + } + return this.resolveRequestWith(broker, latest, responsePayload); + } + + public resolveRequestWith( + broker: BrowserToolsBroker, + request: BrowserAutomationExecuteRequest, + responsePayload: BrowserAutomationExecuteResponse["payload"], ): boolean { return broker.receiveResponse({ type: "browser.automation.execute.response", - payload: responsePayload, + payload: { ...responsePayload, requestId: request.requestId }, }); } } -class FailingDesktopClient implements BrowserToolsDesktopClient { - public readonly id = "desktop-1"; +class FailingBrowserHostClient implements BrowserHostClient { + public readonly id = "host-1"; + public readonly hostKind = "desktop app"; + public readonly supportedCommands = [...BROWSER_AUTOMATION_COMMAND_NAMES]; public sendBrowserAutomationRequest(): void { throw new Error("websocket send failed"); @@ -68,23 +96,23 @@ describe("BrowserToolsBroker", () => { }); }); - test("no capable desktop returns browser_no_desktop", async () => { + test("no connected browser host returns a retryable browser_no_host error", async () => { const broker = createBroker({ enabled: true }); await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({ requestId: "req-1", ok: false, error: { - code: "browser_no_desktop", - message: "No desktop browser automation client is connected.", + code: "browser_no_host", + message: "No browser automation host is connected.", retryable: true, }, }); }); - test("invalid browser requests return structured failures without contacting desktop", async () => { + test("invalid browser requests return structured failures without contacting a host", async () => { const broker = createBroker({ enabled: true }); - const client = new FakeDesktopClient("desktop-1"); + const client = new FakeBrowserHostClient("host-1"); broker.registerClient(client); await expect( @@ -107,9 +135,9 @@ describe("BrowserToolsBroker", () => { expect(broker.getPendingRequestCount()).toBe(0); }); - test("capable fake desktop receives request and returns response", async () => { + test("capable browser host receives request and returns response", async () => { const broker = createBroker({ enabled: true }); - const client = new FakeDesktopClient("desktop-1"); + const client = new FakeBrowserHostClient("host-1"); broker.registerClient(client); const resultPromise = broker.execute({ @@ -165,9 +193,9 @@ describe("BrowserToolsBroker", () => { expect(broker.getPendingRequestCount()).toBe(0); }); - test("desktop receives snapshot requests", async () => { + test("single browser host receives snapshot requests", async () => { const broker = createBroker({ enabled: true }); - const client = new FakeDesktopClient("desktop-1"); + const client = new FakeBrowserHostClient("host-1"); broker.registerClient(client); const resultPromise = broker.execute({ @@ -211,10 +239,404 @@ describe("BrowserToolsBroker", () => { }); }); + test("new tabs target the most recently registered host and tab commands stay with that host", async () => { + const broker = createBroker({ enabled: true }); + const firstHost = new FakeBrowserHostClient("host-1"); + const recentHost = new FakeBrowserHostClient("host-2"); + broker.registerClient(firstHost); + broker.registerClient(recentHost); + + const newTabPromise = broker.execute({ + command: { command: "new_tab", args: { url: "https://example.com" } }, + workspaceId: "workspace-1", + }); + + expect(firstHost.receivedRequests).toEqual([]); + expect(recentHost.receivedRequests).toEqual([ + { + type: "browser.automation.execute.request", + requestId: "req-1", + workspaceId: "workspace-1", + command: { command: "new_tab", args: { url: "https://example.com" } }, + }, + ]); + + recentHost.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "new_tab", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", + }, + }); + + await expect(newTabPromise).resolves.toMatchObject({ + ok: true, + result: { command: "new_tab", browserId: BROWSER_ID }, + }); + + const snapshotPromise = broker.execute({ + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, + workspaceId: "workspace-1", + }); + + expect(firstHost.receivedRequests).toEqual([]); + expect(recentHost.receivedRequests.at(-1)).toEqual({ + type: "browser.automation.execute.request", + requestId: "req-1", + workspaceId: "workspace-1", + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, + }); + + recentHost.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "snapshot", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + elements: [], + }, + }); + + await expect(snapshotPromise).resolves.toMatchObject({ + ok: true, + result: { command: "snapshot", browserId: BROWSER_ID }, + }); + }); + + test("list tabs aggregates all hosts and seeds browser id affinity", async () => { + const broker = createBroker({ enabled: true }); + const firstHost = new FakeBrowserHostClient("host-1"); + const secondHost = new FakeBrowserHostClient("host-2"); + broker.registerClient(firstHost); + broker.registerClient(secondHost); + + const listPromise = broker.execute({ + command: { command: "list_tabs", args: {} }, + workspaceId: "workspace-1", + }); + + expect(firstHost.receivedRequests).toEqual([ + { + type: "browser.automation.execute.request", + requestId: "req-1:host-1", + workspaceId: "workspace-1", + command: { command: "list_tabs", args: {} }, + }, + ]); + expect(secondHost.receivedRequests).toEqual([ + { + type: "browser.automation.execute.request", + requestId: "req-1:host-2", + workspaceId: "workspace-1", + command: { command: "list_tabs", args: {} }, + }, + ]); + + firstHost.resolveLatestWith(broker, { + requestId: "req-1:host-1", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://one.example", + title: "One", + }, + ], + }, + }); + secondHost.resolveLatestWith(broker, { + requestId: "req-1:host-2", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: SECOND_BROWSER_ID, + workspaceId: "workspace-1", + url: "https://two.example", + title: "Two", + }, + ], + }, + }); + + await expect(listPromise).resolves.toEqual({ + requestId: "req-1", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://one.example", + title: "One", + isActive: false, + isLoading: false, + }, + { + browserId: SECOND_BROWSER_ID, + workspaceId: "workspace-1", + url: "https://two.example", + title: "Two", + isActive: false, + isLoading: false, + }, + ], + }, + }); + + const firstSnapshot = broker.execute({ + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, + workspaceId: "workspace-1", + }); + const secondSnapshot = broker.execute({ + command: { command: "snapshot", args: { browserId: SECOND_BROWSER_ID } }, + workspaceId: "workspace-1", + requestId: "req-2", + }); + + expect(firstHost.receivedRequests.at(-1)?.command).toEqual({ + command: "snapshot", + args: { browserId: BROWSER_ID }, + }); + expect(secondHost.receivedRequests.at(-1)?.command).toEqual({ + command: "snapshot", + args: { browserId: SECOND_BROWSER_ID }, + }); + + firstHost.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "snapshot", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://one.example", + title: "One", + elements: [], + }, + }); + secondHost.resolveLatestWith(broker, { + requestId: "req-2", + ok: true, + result: { + command: "snapshot", + browserId: SECOND_BROWSER_ID, + workspaceId: "workspace-1", + url: "https://two.example", + title: "Two", + elements: [], + }, + }); + + await expect(firstSnapshot).resolves.toMatchObject({ + ok: true, + result: { command: "snapshot", browserId: BROWSER_ID }, + }); + await expect(secondSnapshot).resolves.toMatchObject({ + ok: true, + result: { command: "snapshot", browserId: SECOND_BROWSER_ID }, + }); + }); + + test("failed list tabs aggregation does not seed browser id affinity", async () => { + const broker = createBroker({ enabled: true }); + const firstHost = new FakeBrowserHostClient("host-1"); + const secondHost = new FakeBrowserHostClient("host-2"); + broker.registerClient(firstHost); + broker.registerClient(secondHost); + + const listPromise = broker.execute({ + command: { command: "list_tabs", args: {} }, + workspaceId: "workspace-1", + }); + + firstHost.resolveLatestWith(broker, { + requestId: "req-1:host-1", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://one.example", + title: "One", + }, + ], + }, + }); + secondHost.resolveLatestWith(broker, { + requestId: "req-1:host-2", + ok: false, + error: { + code: "browser_timeout", + message: "Host did not answer list_tabs.", + retryable: true, + }, + }); + + await expect(listPromise).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_timeout", + message: "Host did not answer list_tabs.", + retryable: true, + }, + }); + + await expect( + broker.execute({ + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, + workspaceId: "workspace-1", + }), + ).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_tab_not_found", + message: `Browser tab ${BROWSER_ID} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`, + retryable: false, + }, + }); + expect(firstHost.receivedRequests).toHaveLength(1); + expect(secondHost.receivedRequests).toHaveLength(1); + }); + + test("unsupported commands are rejected before sending to the routed host", async () => { + const broker = createBroker({ enabled: true }); + const client = new FakeBrowserHostClient("host-1", { + supportedCommands: ["list_tabs"], + hostKind: "desktop app", + }); + broker.registerClient(client); + + await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_unsupported", + message: 'Browser automation command "snapshot" is not supported by the desktop app.', + retryable: false, + }, + }); + expect(client.receivedRequests).toEqual([]); + }); + + test("unregistering a host strands its browser ids instead of routing them to another host", async () => { + const broker = createBroker({ enabled: true }); + const owner = new FakeBrowserHostClient("host-1"); + const unregisterOwner = broker.registerClient(owner); + + const newTabPromise = broker.execute({ + command: { command: "new_tab", args: {} }, + workspaceId: "workspace-1", + }); + owner.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "new_tab", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", + }, + }); + await newTabPromise; + + unregisterOwner(); + const replacement = new FakeBrowserHostClient("host-2"); + broker.registerClient(replacement); + + await expect( + broker.execute({ + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, + workspaceId: "workspace-1", + }), + ).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_no_host", + message: `The app hosting browser tab ${BROWSER_ID} disconnected.`, + retryable: true, + }, + }); + expect(replacement.receivedRequests).toEqual([]); + }); + + test("reconnecting the same host reclaims its stranded browser ids", async () => { + const broker = createBroker({ enabled: true }); + const owner = new FakeBrowserHostClient("host-1"); + const unregisterOwner = broker.registerClient(owner); + + const newTabPromise = broker.execute({ + command: { command: "new_tab", args: {} }, + workspaceId: "workspace-1", + }); + owner.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "new_tab", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", + }, + }); + await newTabPromise; + + unregisterOwner(); + const reconnectedOwner = new FakeBrowserHostClient("host-1"); + broker.registerClient(reconnectedOwner); + + const snapshotPromise = broker.execute({ + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, + workspaceId: "workspace-1", + }); + + expect(reconnectedOwner.receivedRequests).toEqual([ + { + type: "browser.automation.execute.request", + requestId: "req-1", + workspaceId: "workspace-1", + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, + }, + ]); + reconnectedOwner.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "snapshot", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + elements: [], + }, + }); + + await expect(snapshotPromise).resolves.toMatchObject({ + ok: true, + result: { command: "snapshot", browserId: BROWSER_ID }, + }); + }); + test("timeout resolves browser_timeout and clears pending state", async () => { vi.useFakeTimers(); const broker = createBroker({ enabled: true, timeoutMs: 50 }); - broker.registerClient(new FakeDesktopClient("desktop-1")); + broker.registerClient(new FakeBrowserHostClient("host-1")); const resultPromise = broker.execute({ command: snapshotCommand() }); expect(broker.getPendingRequestCount()).toBe(1); @@ -235,7 +657,7 @@ describe("BrowserToolsBroker", () => { test("disconnect resolves retryable failure and clears pending request", async () => { const broker = createBroker({ enabled: true }); - const client = new FakeDesktopClient("desktop-1"); + const client = new FakeBrowserHostClient("host-1"); const unregister = broker.registerClient(client); const resultPromise = broker.execute({ command: snapshotCommand() }); @@ -247,17 +669,55 @@ describe("BrowserToolsBroker", () => { requestId: "req-1", ok: false, error: { - code: "browser_no_desktop", - message: "The desktop browser automation client disconnected before responding.", + code: "browser_no_host", + message: "The browser automation host disconnected before responding.", retryable: true, }, }); expect(broker.getPendingRequestCount()).toBe(0); }); - test("desktop send failure resolves structured failure and clears pending request", async () => { + test("replacing a host registration resolves pending requests from the old registration", async () => { const broker = createBroker({ enabled: true }); - broker.registerClient(new FailingDesktopClient()); + const oldHost = new FakeBrowserHostClient("host-1"); + const newHost = new FakeBrowserHostClient("host-1"); + broker.registerClient(oldHost); + + const pendingResult = broker.execute({ command: snapshotCommand() }); + expect(oldHost.receivedRequests).toHaveLength(1); + expect(broker.getPendingRequestCount()).toBe(1); + + broker.registerClient(newHost); + + await expect(pendingResult).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_no_host", + message: "The browser automation host disconnected before responding.", + retryable: true, + }, + }); + expect(broker.getPendingRequestCount()).toBe(0); + + const listResult = broker.execute({ command: { command: "list_tabs", args: {} } }); + expect(newHost.receivedRequests).toHaveLength(1); + newHost.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { command: "list_tabs", tabs: [] }, + }); + + await expect(listResult).resolves.toEqual({ + requestId: "req-1", + ok: true, + result: { command: "list_tabs", tabs: [] }, + }); + }); + + test("browser host send failure resolves structured failure and clears pending request", async () => { + const broker = createBroker({ enabled: true }); + broker.registerClient(new FailingBrowserHostClient()); await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({ requestId: "req-1", @@ -273,7 +733,7 @@ describe("BrowserToolsBroker", () => { test("explicit browser failure response propagates typed error", async () => { const broker = createBroker({ enabled: true }); - const client = new FakeDesktopClient("desktop-1"); + const client = new FakeBrowserHostClient("host-1"); broker.registerClient(client); const resultPromise = broker.execute({ command: snapshotCommand() }); @@ -302,7 +762,7 @@ describe("BrowserToolsBroker", () => { test("invalid browser response resolves a structured failure and clears pending state", async () => { const broker = createBroker({ enabled: true }); - const client = new FakeDesktopClient("desktop-1"); + const client = new FakeBrowserHostClient("host-1"); broker.registerClient(client); const resultPromise = broker.execute({ command: snapshotCommand() }); diff --git a/packages/server/src/server/browser-tools/broker.ts b/packages/server/src/server/browser-tools/broker.ts index 83ddc4a69..f938403ff 100644 --- a/packages/server/src/server/browser-tools/broker.ts +++ b/packages/server/src/server/browser-tools/broker.ts @@ -3,14 +3,17 @@ import { BrowserAutomationExecuteRequestSchema, BrowserAutomationExecuteResponseSchema, type BrowserAutomationCommand, + type BrowserAutomationCommandName, type BrowserAutomationExecuteRequest, type BrowserAutomationExecuteResponse, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { browserToolsFailure, type BrowserToolsResponsePayload } from "./errors.js"; import type { BrowserToolsPolicy } from "./policy.js"; -export interface BrowserToolsDesktopClient { +export interface BrowserHostClient { id: string; + hostKind: string; + supportedCommands: readonly BrowserAutomationCommandName[]; sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void | Promise; } @@ -25,10 +28,17 @@ export interface BrowserToolsExecuteInput { interface PendingBrowserToolsRequest { clientId: string; + rememberAffinity: boolean; timeout: ReturnType; resolve: (payload: BrowserToolsResponsePayload) => void; } +interface RegisteredBrowserHost { + client: BrowserHostClient; + registeredAt: number; + supportedCommands: ReadonlySet; +} + export interface BrowserToolsBrokerOptions { policy: BrowserToolsPolicy; defaultTimeoutMs?: number; @@ -41,8 +51,11 @@ export class BrowserToolsBroker { private readonly policy: BrowserToolsPolicy; private readonly defaultTimeoutMs: number; private readonly createRequestId: () => string; - private readonly clients = new Map(); + private readonly clients = new Map(); private readonly pending = new Map(); + private readonly browserHostByBrowserId = new Map(); + private readonly strandedBrowserHostByBrowserId = new Map(); + private registrationSequence = 0; public constructor(options: BrowserToolsBrokerOptions) { this.policy = options.policy; @@ -50,16 +63,31 @@ export class BrowserToolsBroker { this.createRequestId = options.createRequestId ?? (() => `browser_${randomUUID()}`); } - public registerClient(client: BrowserToolsDesktopClient): () => void { - this.clients.set(client.id, client); - return () => this.unregisterClient(client.id); + public registerClient(client: BrowserHostClient): () => void { + this.unregisterClient(client.id); + const registeredAt = ++this.registrationSequence; + this.clients.set(client.id, { + client, + registeredAt, + supportedCommands: new Set(client.supportedCommands), + }); + return () => this.unregisterClient(client.id, registeredAt); } - public unregisterClient(clientId: string): void { - const deleted = this.clients.delete(clientId); - if (!deleted) { + public unregisterClient(clientId: string, registeredAt?: number): void { + const current = this.clients.get(clientId); + if (!current || (registeredAt !== undefined && current.registeredAt !== registeredAt)) { return; } + this.clients.delete(clientId); + + for (const [browserId, ownerClientId] of this.browserHostByBrowserId) { + if (ownerClientId !== clientId) { + continue; + } + this.browserHostByBrowserId.delete(browserId); + this.strandedBrowserHostByBrowserId.set(browserId, clientId); + } for (const [requestId, pending] of this.pending) { if (pending.clientId !== clientId) { @@ -70,8 +98,8 @@ export class BrowserToolsBroker { pending.resolve( browserToolsFailure({ requestId, - code: "browser_no_desktop", - message: "The desktop browser automation client disconnected before responding.", + code: "browser_no_host", + message: "The browser automation host disconnected before responding.", retryable: true, }), ); @@ -97,16 +125,6 @@ export class BrowserToolsBroker { }); } - const client = this.selectClient(); - if (!client) { - return browserToolsFailure({ - requestId, - code: "browser_no_desktop", - message: "No desktop browser automation client is connected.", - retryable: true, - }); - } - const request = BrowserAutomationExecuteRequestSchema.safeParse({ type: "browser.automation.execute.request", requestId, @@ -124,8 +142,29 @@ export class BrowserToolsBroker { }); } + if (request.data.command.command === "list_tabs") { + return this.executeListTabs({ + request: request.data, + timeoutMs: input.timeoutMs ?? this.defaultTimeoutMs, + }); + } + + const host = this.selectHostForCommand(request.data.command, requestId); + if (!host.ok) { + return host.payload; + } + + const unsupported = this.unsupportedCommandFailure({ + host: host.value, + commandName: request.data.command.command, + requestId, + }); + if (unsupported) { + return unsupported; + } + return this.sendRequest({ - client, + host: host.value, request: request.data, timeoutMs: input.timeoutMs ?? this.defaultTimeoutMs, }); @@ -163,23 +202,219 @@ export class BrowserToolsBroker { this.pending.delete(parsed.data.payload.requestId); clearTimeout(pending.timeout); + if (pending.rememberAffinity) { + this.rememberBrowserHostForPayload(pending.clientId, parsed.data.payload); + } pending.resolve(parsed.data.payload); return true; } - private selectClient(): BrowserToolsDesktopClient | null { - for (const client of this.clients.values()) { - return client; - } - return null; - } - - private sendRequest(params: { - client: BrowserToolsDesktopClient; + private async executeListTabs(params: { request: BrowserAutomationExecuteRequest; timeoutMs: number; }): Promise { - const { client, request, timeoutMs } = params; + const hosts = Array.from(this.clients.values()); + if (hosts.length === 0) { + return this.noBrowserHostFailure(params.request.requestId); + } + + for (const host of hosts) { + const unsupported = this.unsupportedCommandFailure({ + host, + commandName: "list_tabs", + requestId: params.request.requestId, + }); + if (unsupported) { + return unsupported; + } + } + + if (hosts.length === 1) { + return this.sendRequest({ + host: hosts[0], + request: params.request, + timeoutMs: params.timeoutMs, + }); + } + + const hostResponses = await Promise.all( + hosts.map(async (host) => ({ + host, + payload: await this.sendRequest({ + host, + request: { + ...params.request, + requestId: `${params.request.requestId}:${host.client.id}`, + }, + rememberAffinity: false, + timeoutMs: params.timeoutMs, + }), + })), + ); + + const failed = hostResponses.find(({ payload }) => !payload.ok); + if (failed) { + return withBrowserToolsRequestId(failed.payload, params.request.requestId); + } + + for (const { host, payload } of hostResponses) { + this.rememberBrowserHostForPayload(host.client.id, payload); + } + + return { + requestId: params.request.requestId, + ok: true, + result: { + command: "list_tabs", + tabs: hostResponses.flatMap(({ payload }) => + payload.ok && payload.result.command === "list_tabs" ? payload.result.tabs : [], + ), + }, + }; + } + + private selectHostForCommand( + command: BrowserAutomationCommand, + requestId: string, + ): + | { ok: true; value: RegisteredBrowserHost } + | { ok: false; payload: BrowserToolsResponsePayload } { + if (command.command === "new_tab") { + const host = this.selectMostRecentlyRegisteredHost(); + return host + ? { ok: true, value: host } + : { ok: false, payload: this.noBrowserHostFailure(requestId) }; + } + + const browserId = getBrowserIdForCommand(command); + if (!browserId) { + const host = this.selectMostRecentlyRegisteredHost(); + return host + ? { ok: true, value: host } + : { ok: false, payload: this.noBrowserHostFailure(requestId) }; + } + + const ownerClientId = this.browserHostByBrowserId.get(browserId); + if (ownerClientId) { + const host = this.clients.get(ownerClientId); + if (host) { + return { ok: true, value: host }; + } + return { + ok: false, + payload: this.strandedBrowserTabFailure({ requestId, browserId }), + }; + } + + const strandedOwnerClientId = this.strandedBrowserHostByBrowserId.get(browserId); + if (strandedOwnerClientId) { + const reconnectedHost = this.clients.get(strandedOwnerClientId); + if (reconnectedHost) { + this.strandedBrowserHostByBrowserId.delete(browserId); + this.browserHostByBrowserId.set(browserId, strandedOwnerClientId); + return { ok: true, value: reconnectedHost }; + } + return { + ok: false, + payload: this.strandedBrowserTabFailure({ requestId, browserId }), + }; + } + + if (this.clients.size === 1) { + const host = this.selectMostRecentlyRegisteredHost(); + if (host) { + return { ok: true, value: host }; + } + } + + if (this.clients.size === 0) { + return { ok: false, payload: this.noBrowserHostFailure(requestId) }; + } + + return { + ok: false, + payload: browserToolsFailure({ + requestId, + code: "browser_tab_not_found", + message: `Browser tab ${browserId} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`, + }), + }; + } + + private selectMostRecentlyRegisteredHost(): RegisteredBrowserHost | null { + let selected: RegisteredBrowserHost | null = null; + for (const host of this.clients.values()) { + selected = host; + } + return selected; + } + + private unsupportedCommandFailure(params: { + host: RegisteredBrowserHost; + commandName: BrowserAutomationCommandName; + requestId: string; + }): BrowserToolsResponsePayload | null { + if (params.host.supportedCommands.has(params.commandName)) { + return null; + } + return browserToolsFailure({ + requestId: params.requestId, + code: "browser_unsupported", + message: `Browser automation command "${params.commandName}" is not supported by the ${describeBrowserHost(params.host)}.`, + }); + } + + private noBrowserHostFailure(requestId: string): BrowserToolsResponsePayload { + return browserToolsFailure({ + requestId, + code: "browser_no_host", + message: "No browser automation host is connected.", + retryable: true, + }); + } + + private strandedBrowserTabFailure(params: { + requestId: string; + browserId: string; + }): BrowserToolsResponsePayload { + return browserToolsFailure({ + requestId: params.requestId, + code: "browser_no_host", + message: `The app hosting browser tab ${params.browserId} disconnected.`, + retryable: true, + }); + } + + private rememberBrowserHostForPayload( + clientId: string, + payload: BrowserToolsResponsePayload, + ): void { + if (!payload.ok) { + return; + } + + if (payload.result.command === "list_tabs") { + for (const tab of payload.result.tabs) { + this.browserHostByBrowserId.set(tab.browserId, clientId); + this.strandedBrowserHostByBrowserId.delete(tab.browserId); + } + return; + } + + if ("browserId" in payload.result) { + this.browserHostByBrowserId.set(payload.result.browserId, clientId); + this.strandedBrowserHostByBrowserId.delete(payload.result.browserId); + } + } + + private sendRequest(params: { + host: RegisteredBrowserHost; + request: BrowserAutomationExecuteRequest; + rememberAffinity?: boolean; + timeoutMs: number; + }): Promise { + const { host, request, timeoutMs } = params; + const client = host.client; return new Promise((resolve) => { const timeout = setTimeout(() => { @@ -198,6 +433,7 @@ export class BrowserToolsBroker { this.pending.set(request.requestId, { clientId: client.id, + rememberAffinity: params.rememberAffinity ?? true, timeout, resolve, }); @@ -225,6 +461,43 @@ export class BrowserToolsBroker { } } +function getBrowserIdForCommand(command: BrowserAutomationCommand): string | null { + switch (command.command) { + case "list_tabs": + case "new_tab": + return null; + case "snapshot": + case "click": + case "fill": + case "wait": + case "type": + case "keypress": + case "navigate": + case "back": + case "forward": + case "reload": + case "screenshot": + case "upload": + case "select": + case "hover": + case "drag": + case "logs": + return command.args.browserId; + } +} + +function describeBrowserHost(host: RegisteredBrowserHost): string { + const hostKind = host.client.hostKind.trim(); + return hostKind || "browser host"; +} + +function withBrowserToolsRequestId( + payload: BrowserToolsResponsePayload, + requestId: string, +): BrowserToolsResponsePayload { + return { ...payload, requestId } as BrowserToolsResponsePayload; +} + function resolveSendFailure(params: { requestId: string; pending: Map; diff --git a/packages/server/src/server/browser-tools/tools.test.ts b/packages/server/src/server/browser-tools/tools.test.ts index c318e64a5..d0848c46e 100644 --- a/packages/server/src/server/browser-tools/tools.test.ts +++ b/packages/server/src/server/browser-tools/tools.test.ts @@ -370,7 +370,7 @@ const brokerErrorCases = [ content: [ { type: "text", - text: "Browser tools are disabled. Enable desktop browser tools on the host, then try again.", + text: "Browser tools are disabled. Enable browser tools on the host, then try again.", }, ], context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" }, @@ -391,7 +391,7 @@ const brokerErrorCases = [ content: [ { type: "text", - text: "The browser did not respond before the timeout. Try again or check the desktop app.", + text: "The browser did not respond before the timeout. Try again or check the browser host.", }, ], context: { diff --git a/packages/server/src/server/browser-tools/tools.ts b/packages/server/src/server/browser-tools/tools.ts index d7b0e0f36..1181bfd56 100644 --- a/packages/server/src/server/browser-tools/tools.ts +++ b/packages/server/src/server/browser-tools/tools.ts @@ -87,7 +87,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "List browser tabs", description: - "List open Paseo desktop browser tabs for this agent's workspace. Use returned browserId values with tab-scoped tools.", + "List open Paseo browser tabs for this agent's workspace across connected browser automation hosts. Use returned browserId values with tab-scoped tools.", inputSchema: {}, outputSchema: BrowserToolOutputSchema, }, @@ -115,7 +115,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Create browser tab", description: - "Create a new Paseo desktop browser tab in this agent's workspace, opened in the background without switching the user's view. Pass an http(s) URL or a scheme-less host URL, which is treated as http; the returned browserId is used by tab-scoped tools.", + "Create a new Paseo browser tab in this agent's workspace on the most recently connected browser automation host, opened in the background without switching the user's view. Pass an http(s) URL or a scheme-less host URL, which is treated as http; the returned browserId is used by tab-scoped tools.", inputSchema: { url: BrowserHttpUrlInputSchema.optional(), }, @@ -145,7 +145,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Snapshot browser page", description: - "Return a model-readable snapshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", + "Return a model-readable snapshot of a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", inputSchema: { browserId: BrowserAutomationBrowserIdSchema, }, @@ -174,7 +174,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Click browser element", description: - "Click an element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", + "Click an element in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", inputSchema: { ref: BrowserRefInputSchema, browserId: BrowserAutomationBrowserIdSchema, @@ -205,7 +205,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Fill browser element", description: - "Fill an input-like element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", + "Fill an input-like element in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", inputSchema: { ref: BrowserRefInputSchema, value: z.string(), @@ -238,7 +238,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Wait for browser condition", description: - "Wait until a Paseo desktop browser tab contains text or reaches a URL fragment. Use browserId from browser_new_tab or browser_list_tabs; waits up to 5s by default on the desktop side.", + "Wait until a Paseo browser tab contains text or reaches a URL fragment. Use browserId from browser_new_tab or browser_list_tabs; waits up to 5s by default on the browser host.", inputSchema: BrowserWaitInputSchema, outputSchema: BrowserToolOutputSchema, }, @@ -335,7 +335,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Navigate browser", description: - "Navigate a Paseo desktop browser tab to a URL. Use browserId from browser_new_tab or browser_list_tabs; pass an http(s) URL or a scheme-less host URL, which is treated as http.", + "Navigate a Paseo browser tab to a URL. Use browserId from browser_new_tab or browser_list_tabs; pass an http(s) URL or a scheme-less host URL, which is treated as http.", inputSchema: { url: BrowserHttpUrlInputSchema, browserId: BrowserAutomationBrowserIdSchema }, outputSchema: BrowserToolOutputSchema, }, @@ -364,21 +364,21 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void command: "back", title: "Browser back", description: - "Go back in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.", + "Go back in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs.", }, { name: "browser_forward", command: "forward", title: "Browser forward", description: - "Go forward in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.", + "Go forward in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs.", }, { name: "browser_reload", command: "reload", title: "Browser reload", description: - "Reload a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.", + "Reload a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs.", }, ] as const) { options.registerTool( @@ -413,7 +413,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Capture browser screenshot", description: - "Capture a PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs. Set fullPage to true to capture the full page.", + "Capture a PNG screenshot of a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs. Set fullPage to true to capture the full page.", inputSchema: { browserId: BrowserAutomationBrowserIdSchema, fullPage: z.boolean().default(false), @@ -444,7 +444,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Upload files in browser", description: - "Set workspace files on a file input in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", + "Set workspace files on a file input in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", inputSchema: { ref: BrowserRefInputSchema, filePaths: z.array(z.string().min(1)).min(1), @@ -478,7 +478,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void command: "hover", title: "Hover browser element", description: - "Hover an element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", + "Hover an element in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", }, ] as const) { options.registerTool( @@ -514,7 +514,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Select browser option", description: - "Set a select element in a Paseo desktop browser tab to a value. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", + "Set a select element in a Paseo browser tab to a value. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", inputSchema: { ref: BrowserRefInputSchema, value: z.string(), @@ -547,7 +547,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Drag browser element", description: - "Drag one element onto another in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", + "Drag one element onto another in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.", inputSchema: { sourceRef: BrowserRefInputSchema, targetRef: BrowserRefInputSchema, @@ -580,7 +580,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Read browser logs", description: - "Read recent console messages and browser performance network entries for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; maxEntries defaults to 50.", + "Read recent console messages and browser performance network entries for a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; maxEntries defaults to 50.", inputSchema: { maxEntries: z.number().int().positive().max(200).optional(), browserId: BrowserAutomationBrowserIdSchema, @@ -886,15 +886,15 @@ function summarizeBrowserError( ): string { switch (error.code) { case "browser_disabled": - return "Browser tools are disabled. Enable desktop browser tools on the host, then try again."; - case "browser_no_desktop": - return "No desktop browser automation client is connected. Open the Paseo desktop app and try again."; + return "Browser tools are disabled. Enable browser tools on the host, then try again."; + case "browser_no_host": + return error.message; case "browser_timeout": - return "The browser did not respond before the timeout. Try again or check the desktop app."; + return "The browser did not respond before the timeout. Try again or check the browser host."; case "screenshot_no_frame": return error.message; case "browser_unsupported": - return "This desktop build does not support that browser automation request yet."; + return error.message; case "browser_stale_ref": return "That browser element reference is stale. Take a new browser snapshot and try again."; default: diff --git a/packages/server/src/server/websocket-server.browser-tools.test.ts b/packages/server/src/server/websocket-server.browser-tools.test.ts index 1ef750c35..dd4b1d1e3 100644 --- a/packages/server/src/server/websocket-server.browser-tools.test.ts +++ b/packages/server/src/server/websocket-server.browser-tools.test.ts @@ -2,9 +2,11 @@ import { createServer, type Server as HTTPServer } from "node:http"; import type { AddressInfo } from "node:net"; import type { + BrowserAutomationCommandName, BrowserAutomationExecuteRequest, BrowserAutomationExecuteResponse, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; import type pino from "pino"; import { afterEach, describe, expect, it } from "vitest"; @@ -26,18 +28,18 @@ import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; interface BrowserToolsDaemonHarness { broker: BrowserToolsBroker; - connectDesktopBrowserClient( - options?: ConnectDesktopBrowserClientOptions, - ): Promise; + connectBrowserHostClient( + options?: ConnectBrowserHostClientOptions, + ): Promise; stop(): Promise; } -interface ConnectDesktopBrowserClientOptions { +interface ConnectBrowserHostClientOptions { clientId?: string; - capabilities?: Partial>; + capabilities?: Record; } -interface DesktopBrowserClientHandle { +interface BrowserHostClientHandle { clientId: string; nextBrowserRequest(): Promise; respondToBrowserRequest(response: BrowserAutomationExecuteResponse): void; @@ -57,15 +59,26 @@ afterEach(async () => { await Promise.all(harnesses.splice(0).map((harness) => harness.stop())); }); +function browserHostCapabilities( + supportedCommands: readonly BrowserAutomationCommandName[] = BROWSER_AUTOMATION_COMMAND_NAMES, +): Record { + return { + [CLIENT_CAPS.browserHost]: { + supportedCommands: [...supportedCommands], + hostKind: "desktop app", + }, + }; +} + describe("WebSocketServer browser tools wiring", () => { it("registers capable clients and dispatches broker requests over the real WebSocket path", async () => { const harness = await startBrowserToolsDaemonHarness(); - const desktop = await harness.connectDesktopBrowserClient(); + const browserHost = await harness.connectBrowserHostClient(); const resultPromise = harness.broker.execute({ command: { command: "list_tabs", args: {} }, }); - const request = await desktop.nextBrowserRequest(); + const request = await browserHost.nextBrowserRequest(); expect(request).toMatchObject({ type: "browser.automation.execute.request", @@ -73,7 +86,7 @@ describe("WebSocketServer browser tools wiring", () => { command: { command: "list_tabs", args: {} }, }); - desktop.respondToBrowserRequest({ + browserHost.respondToBrowserRequest({ type: "browser.automation.execute.response", payload: { requestId: request.requestId, @@ -91,20 +104,20 @@ describe("WebSocketServer browser tools wiring", () => { it("unregisters capable clients on disconnect and clears pending browser commands", async () => { const harness = await startBrowserToolsDaemonHarness(); - const desktop = await harness.connectDesktopBrowserClient(); + const browserHost = await harness.connectBrowserHostClient(); const pendingResult = harness.broker.execute({ command: { command: "list_tabs", args: {} }, }); const pendingExpectation = expect(pendingResult).resolves.toMatchObject({ ok: false, - error: { code: "browser_no_desktop", retryable: true }, + error: { code: "browser_no_host", retryable: true }, }); - await desktop.nextBrowserRequest(); + await browserHost.nextBrowserRequest(); expect(harness.broker.getPendingRequestCount()).toBe(1); - await desktop.disconnect(); + await browserHost.disconnect(); expect(harness.broker.getRegisteredClientCount()).toBe(0); expect(harness.broker.getPendingRequestCount()).toBe(0); @@ -114,28 +127,28 @@ describe("WebSocketServer browser tools wiring", () => { harness.broker.execute({ command: { command: "list_tabs", args: {} } }), ).resolves.toMatchObject({ ok: false, - error: { code: "browser_no_desktop" }, + error: { code: "browser_no_host" }, }); }); - it("keeps browser automation registered when a desktop browser client resumes", async () => { + it("keeps browser automation registered when a browser host client resumes", async () => { const harness = await startBrowserToolsDaemonHarness(); - const clientId = "desktop-client-1"; - await harness.connectDesktopBrowserClient({ + const clientId = "browser-host-client-1"; + await harness.connectBrowserHostClient({ clientId, - capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true }, + capabilities: browserHostCapabilities(), }); - const resumedDesktop = await harness.connectDesktopBrowserClient({ + const resumedBrowserHost = await harness.connectBrowserHostClient({ clientId, - capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true }, + capabilities: browserHostCapabilities(), }); const resultPromise = harness.broker.execute({ command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } }, }); - const request = await resumedDesktop.nextBrowserRequest(); - resumedDesktop.respondToBrowserRequest({ + const request = await resumedBrowserHost.nextBrowserRequest(); + resumedBrowserHost.respondToBrowserRequest({ type: "browser.automation.execute.response", payload: { requestId: request.requestId, @@ -149,6 +162,33 @@ describe("WebSocketServer browser tools wiring", () => { result: { command: "click", browserId: BROWSER_ID, ref: "@e1" }, }); }); + + it("clears pending browser commands when a browser host changes capabilities", async () => { + const harness = await startBrowserToolsDaemonHarness(); + const clientId = "browser-host-client-1"; + const browserHost = await harness.connectBrowserHostClient({ + clientId, + capabilities: browserHostCapabilities(), + }); + + const pendingResult = harness.broker.execute({ + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, + }); + await browserHost.nextBrowserRequest(); + expect(harness.broker.getPendingRequestCount()).toBe(1); + + await harness.connectBrowserHostClient({ + clientId, + capabilities: browserHostCapabilities(["list_tabs"]), + }); + + await expect(pendingResult).resolves.toMatchObject({ + ok: false, + error: { code: "browser_no_host", retryable: true }, + }); + expect(harness.broker.getRegisteredClientCount()).toBe(1); + expect(harness.broker.getPendingRequestCount()).toBe(0); + }); }); async function startBrowserToolsDaemonHarness(): Promise { @@ -162,7 +202,7 @@ async function startBrowserToolsDaemonHarness(): Promise | null, -): boolean { - return capabilities?.[CLIENT_CAPS.desktopBrowserAutomation] === true; +): BrowserAutomationHostCapability | null { + const parsed = BrowserAutomationHostCapabilitySchema.safeParse( + capabilities?.[CLIENT_CAPS.browserHost], + ); + return parsed.success ? parsed.data : null; } interface WebSocketLike { @@ -333,6 +340,7 @@ interface SessionConnection { } interface BrowserToolsRegistration { + capabilitySignature: string; unregister: () => void; } @@ -1412,22 +1420,31 @@ export class VoiceAssistantWebSocketServer { if (!this.browserToolsBroker) { return; } - if (!hasDesktopBrowserAutomationCapability(connection.clientCapabilities)) { + const browserHostCapability = getBrowserHostCapability(connection.clientCapabilities); + if (!browserHostCapability) { this.unregisterBrowserToolsClient(connection.clientId); return; } + const capabilitySignature = JSON.stringify(browserHostCapability); const existing = this.browserToolsRegistrations.get(connection.clientId); - if (existing) { + if (existing?.capabilitySignature === capabilitySignature) { return; } + if (existing) { + this.browserToolsRegistrations.delete(connection.clientId); + existing.unregister(); + } const unregister = this.browserToolsBroker.registerClient({ id: connection.clientId, + hostKind: browserHostCapability.hostKind, + supportedCommands: browserHostCapability.supportedCommands, sendBrowserAutomationRequest: (request) => { this.sendToConnection(connection, wrapSessionMessage(request)); }, }); this.browserToolsRegistrations.set(connection.clientId, { + capabilitySignature, unregister, }); }